-- Logs begin at Fri 2022-07-01 08:45:11 UTC, end at Fri 2022-07-01 11:20:26 UTC. -- Jul 01 09:18:10.243165 dv-neutron-848393-1-hyperv systemd[1]: Started Devstack devstack@q-l3.service. Jul 01 09:18:13.419594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.common.config [-] Logging enabled! Jul 01 09:18:13.420199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.common.config [-] /usr/local/bin/neutron-l3-agent version 20.1.0.dev388 Jul 01 09:18:13.420604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.config [-] command line: /usr/local/bin/neutron-l3-agent --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/l3_agent.ini {{(pid=106231) setup_logging /opt/stack/neutron/neutron/common/config.py:120}} Jul 01 09:18:13.852920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_create 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.853493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_update 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.853903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_delete 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.855242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_create 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.855675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_update 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.856104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_delete 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.908953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_create 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.909469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_update 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.909883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_delete 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.921177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_create 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.921651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_update 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.922063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_delete 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.944828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_create 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.945364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_update 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.945773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_delete 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.981773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_create 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.982283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_update 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:13.982694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > rbac-policy before_delete 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:14.118371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Subscribe: > router after_delete 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:14.212456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [None req-874c3fcc-699f-49eb-80cf-8f0d565f9d2c None None] Time-cost: call 48562400-7b17-454a-ae09-4f3ad4ce7b3e function get_host_ha_router_count start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:18:14.465386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [None req-874c3fcc-699f-49eb-80cf-8f0d565f9d2c None None] Time-cost: call 48562400-7b17-454a-ae09-4f3ad4ce7b3e function get_host_ha_router_count took 0.253s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:18:14.466644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [None req-874c3fcc-699f-49eb-80cf-8f0d565f9d2c None None] Agent HA routers count 0 Jul 01 09:18:14.467717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.agent_extensions_manager [None req-874c3fcc-699f-49eb-80cf-8f0d565f9d2c None None] Loaded agent extensions: [] Jul 01 09:18:14.468471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [None req-874c3fcc-699f-49eb-80cf-8f0d565f9d2c None None] Subscribe: router after_create 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:14.468985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [None req-874c3fcc-699f-49eb-80cf-8f0d565f9d2c None None] Subscribe: router after_update 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:14.469539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [None req-874c3fcc-699f-49eb-80cf-8f0d565f9d2c None None] Subscribe: router before_delete 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:14.478786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] Subscribe: router before_create 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:14.479887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] Subscribe: router after_update 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:14.480378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] Subscribe: router after_delete 55550000 {{(pid=106231) subscribe /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:45}} Jul 01 09:18:14.490659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ******************************************************************************** {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2589}} Jul 01 09:18:14.491029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] Configuration options gathered from: {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2590}} Jul 01 09:18:14.491401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] command line args: ['--config-file', '/etc/neutron/neutron.conf', '--config-file', '/etc/neutron/l3_agent.ini'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2591}} Jul 01 09:18:14.492250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] config files: ['/etc/neutron/neutron.conf', '/etc/neutron/l3_agent.ini'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2592}} Jul 01 09:18:14.492580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ================================================================================ {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2594}} Jul 01 09:18:14.493069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] agent_down_time = 75 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.493470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] agent_mode = legacy {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.493853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] allow_bulk = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.494270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] api_extensions_path = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.495262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] api_paste_config = api-paste.ini {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.495682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] api_workers = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.496123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] auth_strategy = keystone {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.496505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] backlog = 4096 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.496896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] base_mac = fa:16:3e:00:00:00 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.497334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] bind_host = 0.0.0.0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.497737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] bind_port = 9696 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.498161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] cleanup_on_shutdown = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.498483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] client_socket_timeout = 900 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.498867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] config_dir = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.499195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] config_file = ['/etc/neutron/neutron.conf', '/etc/neutron/l3_agent.ini'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.499693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] config_source = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.500140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] control_exchange = neutron {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.500558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] core_plugin = ml2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.501630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] debug = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.502090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] default_availability_zones = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.502784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] default_log_levels = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', 'oslo.messaging=INFO', 'oslo_messaging=INFO', 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN', 'urllib3.connectionpool=WARN', 'websocket=WARN', 'requests.packages.urllib3.util.retry=WARN', 'urllib3.util.retry=WARN', 'keystonemiddleware=WARN', 'routes.middleware=WARN', 'stevedore=WARN', 'taskflow=WARN', 'keystoneauth=WARN', 'oslo.cache=INFO', 'oslo_policy=INFO', 'dogpile.core.dogpile=INFO', 'OFPHandler=INFO', 'OfctlService=INFO', 'os_ken.base.app_manager=INFO', 'os_ken.controller.controller=INFO'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.504227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] dhcp_agent_notification = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.504673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] dhcp_lease_duration = 86400 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.505070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] dhcp_load_type = networks {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.505463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] dns_domain = openstacklocal {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.505845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] enable_metadata_proxy = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.506222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] enable_new_agents = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.506653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] enable_traditional_dhcp = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.507031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] executor_thread_pool_size = 64 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.507478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] external_dns_driver = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.507900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] external_ingress_mark = 0x2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.508399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] external_pids = /opt/stack/data/neutron/external/pids {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.508806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] filter_validation = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.509239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] global_physnet_mtu = 1500 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.509736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ha_confs_path = /opt/stack/data/neutron/ha_confs {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.510093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ha_keepalived_state_change_server_threads = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.510498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ha_vrrp_advert_int = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.510929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ha_vrrp_auth_password = **** {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.511346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ha_vrrp_auth_type = PASS {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.511794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ha_vrrp_health_check_interval = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.512198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] handle_internal_only_routers = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.512765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] host = dv-neutron-848393-1-hyperv {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.513175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] http_retries = 3 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.513593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] instance_format = [instance: %(uuid)s] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.514010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] instance_uuid_format = [instance: %(uuid)s] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.514446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] interface_driver = openvswitch {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.514907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ipam_driver = internal {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.515325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ipv6_gateway = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.515760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ipv6_pd_enabled = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.516166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] keepalived_use_no_track = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.516757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] log_config_append = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.517223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] log_date_format = %Y-%m-%d %H:%M:%S {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.517660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] log_dir = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.518069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] log_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.518486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] log_rotate_interval = 1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.518950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] log_rotate_interval_type = days {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.519417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] log_rotation_type = none {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.519857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] logging_context_format_string = %(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.520223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] logging_debug_format_suffix = {{(pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d}} {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.520587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] logging_default_format_string = %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.521005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] logging_exception_prefix = ERROR %(name)s %(instance)s {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.521424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.521920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] max_dns_nameservers = 5 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.522270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] max_header_line = 16384 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.522675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] max_logfile_count = 30 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.523130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] max_logfile_size_mb = 200 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.523598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] max_rtr_adv_interval = 100 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.524005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] max_subnet_host_routes = 20 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.524439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] metadata_access_mark = 0x1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.524844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] metadata_port = 9697 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.525261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] metadata_proxy_group = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.525749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] metadata_proxy_socket = /opt/stack/data/neutron/metadata_proxy {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.526165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] metadata_proxy_user = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.526569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] min_rtr_adv_interval = 30 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.526981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] network_link_prefix = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.527423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] notify_nova_on_port_data_changes = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.527856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] notify_nova_on_port_status_changes = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.528275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ovs_integration_bridge = br-int {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.528687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ovs_use_veth = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.529102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] pagination_max_limit = -1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.529597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] pd_confs = /opt/stack/data/neutron/pd {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.530028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] periodic_fuzzy_delay = 5 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.530443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] periodic_interval = 40 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.530858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] prefix_delegation_driver = dibbler {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.531294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] publish_errors = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.531812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ra_confs = /opt/stack/data/neutron/ra {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.532519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] radvd_user = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.533014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] rate_limit_burst = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.533434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] rate_limit_except_level = CRITICAL {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.533839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] rate_limit_interval = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.534252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] retry_until_window = 30 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.534653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] rpc_ping_enabled = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.535065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] rpc_resources_processing_step = 20 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.535524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] rpc_response_max_timeout = 600 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.535930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] rpc_response_timeout = 60 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.536361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] rpc_state_report_workers = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.536759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] rpc_workers = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.537161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] run_external_periodic_tasks = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.537563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] send_events_interval = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.538012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] service_plugins = ['router', 'neutron.services.metering.metering_plugin.MeteringPlugin'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.538432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] setproctitle = on {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.538783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] state_path = /opt/stack/data/neutron {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.539241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] syslog_log_facility = LOG_USER {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.539685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] tcp_keepidle = 600 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.540098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] transport_url = **** {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.540523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] use_eventlog = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.540924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] use_journal = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.541333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] use_json = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.541685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] use_ssl = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.542084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] use_stderr = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.542495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] use_syslog = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.543052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] vendor_pen = 8888 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.543887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] vlan_transparent = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.544305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] watch_log_file = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.544717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] wsgi_default_pool_size = 100 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.545125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] wsgi_keep_alive = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.545546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.545972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] wsgi_server_debug = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.546405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_concurrency.disable_process_locking = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.546835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_concurrency.lock_path = /opt/stack/data/neutron/lock {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.547321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] profiler.connection_string = messaging:// {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.548295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] profiler.enabled = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.548763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] profiler.es_doc_type = notification {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.549230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] profiler.es_scroll_size = 10000 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.549655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] profiler.es_scroll_time = 2m {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.550076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] profiler.filter_error_trace = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.550508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] profiler.hmac_keys = SECRET_KEY {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.550931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] profiler.sentinel_service_name = mymaster {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.551381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] profiler.socket_timeout = 0.1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.551801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] profiler.trace_sqlalchemy = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.552255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_policy.enforce_new_defaults = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.552685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_policy.enforce_scope = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.553110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_policy.policy_default_rule = default {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.553551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_policy.policy_dirs = ['policy.d'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.553990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_policy.policy_file = /etc/neutron/policy.json {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.554414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_policy.remote_content_type = application/x-www-form-urlencoded {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.554820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_policy.remote_ssl_ca_crt_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.555251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_policy.remote_ssl_client_crt_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.555694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_policy.remote_ssl_client_key_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.556106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_policy.remote_ssl_verify_server_crt = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.556543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_metrics.metrics_buffer_size = 1000 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.556953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_metrics.metrics_enabled = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.557377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_metrics.metrics_process_name = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.557797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_metrics.metrics_socket_file = /var/tmp/metrics_collector.sock {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.558209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_metrics.metrics_thread_stop_timeout = 10 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.558639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_middleware.http_basic_auth_user_file = /etc/htpasswd {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.559099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep.capabilities = [21, 12, 1, 2, 19] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.560083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep.group = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.560500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep.helper_command = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.560920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep.logger_name = oslo_privsep.daemon {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.561338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep.thread_pool_size = 4 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.561895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep.user = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.562344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_dhcp_release.capabilities = [21, 12] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.562760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_dhcp_release.group = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.564006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_dhcp_release.helper_command = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.564429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_dhcp_release.logger_name = oslo_privsep.daemon {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.564843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_dhcp_release.thread_pool_size = 4 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.565271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_dhcp_release.user = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.565690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_ovs_vsctl.capabilities = [21, 12] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.566103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_ovs_vsctl.group = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.566507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_ovs_vsctl.helper_command = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.566922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_ovs_vsctl.logger_name = oslo_privsep.daemon {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.567339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_ovs_vsctl.thread_pool_size = 4 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.567781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_ovs_vsctl.user = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.568210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_namespace.capabilities = [21] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.568628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_namespace.group = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.569037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_namespace.helper_command = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.569459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_namespace.logger_name = oslo_privsep.daemon {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.569882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_namespace.thread_pool_size = 4 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.570297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_namespace.user = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.570712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_conntrack.capabilities = [12] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.571125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_conntrack.group = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.571634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_conntrack.helper_command = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.572068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_conntrack.logger_name = oslo_privsep.daemon {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.572475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_conntrack.thread_pool_size = 4 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.572876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_conntrack.user = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.573375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_link.capabilities = [12, 21] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.573778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_link.group = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.574185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_link.helper_command = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.574614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_link.logger_name = oslo_privsep.daemon {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.575038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_link.thread_pool_size = 4 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.575463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] privsep_link.user = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.575901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.arp_responder = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.576336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.availability_zone = nova {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.576748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.baremetal_smartnic = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.577188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.check_child_processes_action = respawn {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.577554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.check_child_processes_interval = 60 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.577963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.comment_iptables_rules = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.578374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.debug_iptables_rules = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.578792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.dont_fragment = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.579403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.drop_flows_on_start = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.579895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.enable_distributed_routing = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.580311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.explicitly_egress_direct = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.580748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.kill_scripts_path = /etc/neutron/kill_scripts/ {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.581161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.l2_population = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.581582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.log_agent_heartbeats = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.582006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.minimize_polling = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.582419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.ovsdb_monitor_respawn_interval = 30 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.582837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.report_interval = 30.0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.583257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.root_helper = sudo /usr/local/bin/neutron-rootwrap /etc/neutron/rootwrap.conf {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.583729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.root_helper_daemon = sudo /usr/local/bin/neutron-rootwrap-daemon /etc/neutron/rootwrap.conf {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.584143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.tunnel_csum = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.584657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.tunnel_types = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.585070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.use_helper_for_ns_read = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.585480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.veth_mtu = 9000 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.585892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] AGENT.vxlan_udp_port = 4789 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.586320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] service_providers.service_provider = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.586748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] nova.auth_section = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.587183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] nova.auth_type = password {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.587668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] nova.cafile = /opt/stack/data/ca-bundle.pem {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.588077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] nova.certfile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.588484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] nova.collect_timing = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.588908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] nova.endpoint_type = public {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.589317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] nova.insecure = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.589738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] nova.keyfile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.590158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] nova.region_name = RegionOne {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.590574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] nova.split_loggers = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.590980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] nova.timeout = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.591410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] placement.auth_section = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.591835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] placement.auth_type = password {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.592278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] placement.cafile = /opt/stack/data/ca-bundle.pem {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.592683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] placement.certfile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.593092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] placement.collect_timing = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.593512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] placement.endpoint_type = public {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.593917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] placement.insecure = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.594321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] placement.keyfile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.594797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] placement.region_name = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.595387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] placement.split_loggers = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.595795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] placement.timeout = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.596214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.auth_section = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.596623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.auth_type = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.597029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.cafile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.597452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.certfile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.597863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.collect_timing = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.598268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.connect_retries = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.598686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.connect_retry_delay = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.599106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.enable_notifications = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.599666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.endpoint_override = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.600089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.insecure = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.600498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.interface = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.600901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.keyfile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.601309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.max_version = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.601720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.min_version = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.602144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.region_name = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.602550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.service_name = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.602967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.service_type = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.603407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.split_loggers = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.603862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.status_code_retries = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.604264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.status_code_retry_delay = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.604680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.timeout = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.605083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.valid_interfaces = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.605484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ironic.version = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.605917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.bridge_mac_table_size = 50000 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.606328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.bridge_mappings = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.606761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.datapath_type = system {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.607183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.igmp_snooping_enable = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.607624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.int_peer_patch_port = patch-tun {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.608069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.integration_bridge = br-int {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.608488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.local_ip = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.608898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.of_connect_timeout = 300 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.609305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.of_inactivity_probe = 10 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.609750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.of_listen_address = 127.0.0.1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.610716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.of_listen_port = 6633 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.611461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.of_request_timeout = 300 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.611893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.openflow_processed_per_port = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.612338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.ovsdb_connection = tcp:127.0.0.1:6640 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.612763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.ovsdb_debug = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.613189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.ovsdb_timeout = 10 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.613615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.resource_provider_bandwidths = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.614023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.resource_provider_default_hypervisor = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.614442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.resource_provider_hypervisors = {} {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.614908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.resource_provider_inventory_defaults = {'allocation_ratio': 1.0, 'min_unit': 1, 'step_size': 1, 'reserved': 0} {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.615457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.resource_provider_packet_processing_inventory_defaults = {'allocation_ratio': 1.0, 'min_unit': 1, 'step_size': 1, 'reserved': 0} {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.615905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.resource_provider_packet_processing_with_direction = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.616323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.resource_provider_packet_processing_without_direction = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.616733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.ssl_ca_cert_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.617153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.ssl_cert_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.617570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.ssl_key_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.618008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.tun_peer_patch_port = patch-int {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.618443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.tunnel_bridge = br-tun {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.618865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] OVS.vhostuser_socket_dir = /var/run/openvswitch {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.619247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.amqp_auto_delete = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.619627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.amqp_durable_queues = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.620073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.conn_pool_min_size = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.620510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.conn_pool_ttl = 1200 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.620866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.direct_mandatory_flag = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.621223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.enable_cancel_on_failover = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.621578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.heartbeat_in_pthread = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.621934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.heartbeat_rate = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.622297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.heartbeat_timeout_threshold = 60 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.622666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.kombu_compression = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.623028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.kombu_failover_strategy = round-robin {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.623411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout = 60 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.623784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.kombu_reconnect_delay = 1.0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.624134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.rabbit_ha_queues = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.624492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.rabbit_interval_max = 30 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.624851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.rabbit_login_method = AMQPLAIN {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.625209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.rabbit_qos_prefetch_count = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.625560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.rabbit_quorum_queue = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.626498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.rabbit_retry_backoff = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.626939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.rabbit_retry_interval = 1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.627296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.rabbit_transient_queues_ttl = 1800 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.627764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.rpc_conn_pool_size = 30 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.628119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.ssl = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.628548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.ssl_ca_file = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.628975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.ssl_cert_file = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.629327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.ssl_enforce_fips_mode = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.629771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.ssl_key_file = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.630775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_rabbit.ssl_version = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.631220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_notifications.driver = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.631649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_notifications.retry = -1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.632097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_notifications.topics = ['notifications'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.632508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_messaging_notifications.transport_url = **** {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.633202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] agent.extensions = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.633635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] oslo_versionedobjects.fatal_exception_format_errors = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.634094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] QUOTAS.default_quota = -1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.634530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] QUOTAS.quota_driver = neutron.db.quota.driver_nolock.DbQuotaNoLockDriver {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.634959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] QUOTAS.quota_network = 100 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.635415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] QUOTAS.quota_port = 500 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.635854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] QUOTAS.quota_rbac_policy = 10 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.636278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] QUOTAS.quota_security_group = 100 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.638200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] QUOTAS.quota_security_group_rule = 100 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.638655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] QUOTAS.quota_subnet = 100 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.639087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] QUOTAS.track_quota_usage = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.639547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] DHCP.dhcp_rebinding_time = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.640055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] DHCP.dhcp_renewal_time = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.640472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] DHCP.enable_ipv6 = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.640902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] LOCAL_IP.static_nat = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.641255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.wsgi [-] ******************************************************************************** {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2613}} Jul 01 09:18:14.643607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO eventlet.wsgi.server [-] (106231) wsgi starting up on http:/opt/stack/data/neutron/keepalived-state-change Jul 01 09:18:14.669952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] Acquired lock "singleton_lock" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:18:14.670657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] Releasing lock "singleton_lock" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:18:14.671432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] Full set of CONF: {{(pid=106231) _wait_for_exit_or_signal /usr/local/lib/python3.8/dist-packages/oslo_service/service.py:362}} Jul 01 09:18:14.671808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ******************************************************************************** {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2589}} Jul 01 09:18:14.672173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] Configuration options gathered from: {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2590}} Jul 01 09:18:14.672544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] command line args: ['--config-file', '/etc/neutron/neutron.conf', '--config-file', '/etc/neutron/l3_agent.ini'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2591}} Jul 01 09:18:14.673506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] config files: ['/etc/neutron/neutron.conf', '/etc/neutron/l3_agent.ini'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2592}} Jul 01 09:18:14.673951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ================================================================================ {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2594}} Jul 01 09:18:14.674424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] agent_down_time = 75 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.674911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] agent_mode = legacy {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.675348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] allow_bulk = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.675911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] api_extensions_path = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.676336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] api_paste_config = api-paste.ini {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.676761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] api_workers = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.677182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] auth_strategy = keystone {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.677543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] backdoor_port = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.677898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] backdoor_socket = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.678304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] backlog = 4096 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.678725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] base_mac = fa:16:3e:00:00:00 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.679176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] bind_host = 0.0.0.0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.679644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] bind_port = 9696 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.680075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] cleanup_on_shutdown = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.680476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] client_socket_timeout = 900 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.680989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] config_dir = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.681346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] config_file = ['/etc/neutron/neutron.conf', '/etc/neutron/l3_agent.ini'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.681863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] config_source = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.682286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] control_exchange = neutron {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.682705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] core_plugin = ml2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.683122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] debug = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.683561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] default_availability_zones = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.684141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] default_log_levels = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', 'oslo.messaging=INFO', 'oslo_messaging=INFO', 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN', 'urllib3.connectionpool=WARN', 'websocket=WARN', 'requests.packages.urllib3.util.retry=WARN', 'urllib3.util.retry=WARN', 'keystonemiddleware=WARN', 'routes.middleware=WARN', 'stevedore=WARN', 'taskflow=WARN', 'keystoneauth=WARN', 'oslo.cache=INFO', 'oslo_policy=INFO', 'dogpile.core.dogpile=INFO', 'OFPHandler=INFO', 'OfctlService=INFO', 'os_ken.base.app_manager=INFO', 'os_ken.controller.controller=INFO'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.684581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] dhcp_agent_notification = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.684987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] dhcp_lease_duration = 86400 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.685411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] dhcp_load_type = networks {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.685830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] dns_domain = openstacklocal {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.686237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] enable_metadata_proxy = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.686640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] enable_new_agents = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.687043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] enable_traditional_dhcp = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.687480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] executor_thread_pool_size = 64 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.688142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] external_dns_driver = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.689327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] external_ingress_mark = 0x2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.690074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] external_pids = /opt/stack/data/neutron/external/pids {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.690497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] filter_validation = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.691055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] global_physnet_mtu = 1500 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.691499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] graceful_shutdown_timeout = 60 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.692128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ha_confs_path = /opt/stack/data/neutron/ha_confs {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.692546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ha_keepalived_state_change_server_threads = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.692959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ha_vrrp_advert_int = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.693366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ha_vrrp_auth_password = **** {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.693794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ha_vrrp_auth_type = PASS {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.694203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ha_vrrp_health_check_interval = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.694612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] handle_internal_only_routers = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.695124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] host = dv-neutron-848393-1-hyperv {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.695607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] http_retries = 3 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.696035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] instance_format = [instance: %(uuid)s] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.696462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] instance_uuid_format = [instance: %(uuid)s] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.696882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] interface_driver = openvswitch {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.697308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ipam_driver = internal {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.697733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ipv6_gateway = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.698141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ipv6_pd_enabled = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.699170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] keepalived_use_no_track = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.699628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] log_config_append = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.700266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] log_date_format = %Y-%m-%d %H:%M:%S {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.700694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] log_dir = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.701236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] log_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.701597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] log_options = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.702146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] log_rotate_interval = 1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.702572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] log_rotate_interval_type = days {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.703000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] log_rotation_type = none {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.703377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] logging_context_format_string = %(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.703746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] logging_debug_format_suffix = {{(pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d}} {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.704272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] logging_default_format_string = %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.704697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] logging_exception_prefix = ERROR %(name)s %(instance)s {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.705125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.705555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] max_dns_nameservers = 5 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.705945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] max_header_line = 16384 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.706353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] max_logfile_count = 30 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.706757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] max_logfile_size_mb = 200 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.707166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] max_rtr_adv_interval = 100 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.707595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] max_subnet_host_routes = 20 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.708067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] metadata_access_mark = 0x1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.708478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] metadata_port = 9697 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.708897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] metadata_proxy_group = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.709399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] metadata_proxy_socket = /opt/stack/data/neutron/metadata_proxy {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.709837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] metadata_proxy_user = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.710250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] min_rtr_adv_interval = 30 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.710643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] network_link_prefix = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.711061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] notify_nova_on_port_data_changes = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.711667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] notify_nova_on_port_status_changes = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.712196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ovs_integration_bridge = br-int {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.712614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ovs_use_veth = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.713034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] pagination_max_limit = -1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.713532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] pd_confs = /opt/stack/data/neutron/pd {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.713939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] periodic_fuzzy_delay = 5 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.714362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] periodic_interval = 40 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.714786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] prefix_delegation_driver = dibbler {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.715193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] publish_errors = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.715754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ra_confs = /opt/stack/data/neutron/ra {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.716172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] radvd_user = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.716578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] rate_limit_burst = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.716993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] rate_limit_except_level = CRITICAL {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.717395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] rate_limit_interval = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.717806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] retry_until_window = 30 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.718212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] rpc_ping_enabled = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.718617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] rpc_resources_processing_step = 20 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.719034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] rpc_response_max_timeout = 600 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.719497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] rpc_response_timeout = 60 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.720018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] rpc_state_report_workers = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.720425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] rpc_workers = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.720828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] run_external_periodic_tasks = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.721319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] send_events_interval = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.721766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] service_plugins = ['router', 'neutron.services.metering.metering_plugin.MeteringPlugin'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.722277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] setproctitle = on {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.722635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] state_path = /opt/stack/data/neutron {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.723056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] syslog_log_facility = LOG_USER {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.723487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] tcp_keepidle = 600 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.723928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] transport_url = **** {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.724329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] use_eventlog = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.724732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] use_journal = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.725141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] use_json = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.725539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] use_ssl = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.725942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] use_stderr = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.726349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] use_syslog = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.726767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] vendor_pen = 8888 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.727171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] vlan_transparent = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.727590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] watch_log_file = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.727994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] wsgi_default_pool_size = 100 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.728406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] wsgi_keep_alive = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.728824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.729227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] wsgi_server_debug = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Jul 01 09:18:14.729671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_concurrency.disable_process_locking = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.730111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_concurrency.lock_path = /opt/stack/data/neutron/lock {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.730548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] profiler.connection_string = messaging:// {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.730968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] profiler.enabled = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.731516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] profiler.es_doc_type = notification {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.731933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] profiler.es_scroll_size = 10000 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.732449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] profiler.es_scroll_time = 2m {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.732856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] profiler.filter_error_trace = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.733299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] profiler.hmac_keys = SECRET_KEY {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.733724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] profiler.sentinel_service_name = mymaster {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.734146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] profiler.socket_timeout = 0.1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.734558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] profiler.trace_sqlalchemy = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.734992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_policy.enforce_new_defaults = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.736086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_policy.enforce_scope = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.736522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_policy.policy_default_rule = default {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.737008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_policy.policy_dirs = ['policy.d'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.737446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_policy.policy_file = /etc/neutron/policy.json {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.737877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_policy.remote_content_type = application/x-www-form-urlencoded {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.738288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_policy.remote_ssl_ca_crt_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.738865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_policy.remote_ssl_client_crt_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.739276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_policy.remote_ssl_client_key_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.739714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_policy.remote_ssl_verify_server_crt = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.740197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_metrics.metrics_buffer_size = 1000 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.740609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_metrics.metrics_enabled = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.741033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_metrics.metrics_process_name = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.741458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_metrics.metrics_socket_file = /var/tmp/metrics_collector.sock {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.741980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_metrics.metrics_thread_stop_timeout = 10 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.742520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_middleware.http_basic_auth_user_file = /etc/htpasswd {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.742956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep.capabilities = [21, 12, 1, 2, 19] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.743655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep.group = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.744072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep.helper_command = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.744494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep.logger_name = oslo_privsep.daemon {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.744910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep.thread_pool_size = 4 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.745327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep.user = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_dhcp_release.capabilities = [21, 12] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.746162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_dhcp_release.group = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.746580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_dhcp_release.helper_command = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.747275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_dhcp_release.logger_name = oslo_privsep.daemon {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.747725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_dhcp_release.thread_pool_size = 4 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.748132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_dhcp_release.user = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.748575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_ovs_vsctl.capabilities = [21, 12] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.748993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_ovs_vsctl.group = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.749402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_ovs_vsctl.helper_command = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.749818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_ovs_vsctl.logger_name = oslo_privsep.daemon {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.750238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_ovs_vsctl.thread_pool_size = 4 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.750641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_ovs_vsctl.user = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.751114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_namespace.capabilities = [21] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.751547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_namespace.group = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.752067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_namespace.helper_command = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.752585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_namespace.logger_name = oslo_privsep.daemon {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.753499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_namespace.thread_pool_size = 4 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.753919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_namespace.user = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.754344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_conntrack.capabilities = [12] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.754752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_conntrack.group = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.755165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_conntrack.helper_command = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.755699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_conntrack.logger_name = oslo_privsep.daemon {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.756106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_conntrack.thread_pool_size = 4 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.756511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_conntrack.user = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.756938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_link.capabilities = [12, 21] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.757347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_link.group = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.757753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_link.helper_command = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.758171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_link.logger_name = oslo_privsep.daemon {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.758599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_link.thread_pool_size = 4 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.759005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] privsep_link.user = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.759458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.arp_responder = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.759891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.availability_zone = nova {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.760309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.baremetal_smartnic = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.760741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.check_child_processes_action = respawn {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.761151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.check_child_processes_interval = 60 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.761571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.comment_iptables_rules = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.762214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.debug_iptables_rules = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.762735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.dont_fragment = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.763151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.drop_flows_on_start = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.763596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.enable_distributed_routing = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.764011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.explicitly_egress_direct = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.764440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.kill_scripts_path = /etc/neutron/kill_scripts/ {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.764860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.l2_population = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.765274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.log_agent_heartbeats = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.765684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.minimize_polling = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.766095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.ovsdb_monitor_respawn_interval = 30 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.766541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.report_interval = 30.0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.767075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.root_helper = sudo /usr/local/bin/neutron-rootwrap /etc/neutron/rootwrap.conf {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.767421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.root_helper_daemon = sudo /usr/local/bin/neutron-rootwrap-daemon /etc/neutron/rootwrap.conf {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.767846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.tunnel_csum = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.768284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.tunnel_types = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.768698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.use_helper_for_ns_read = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.769122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.veth_mtu = 9000 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.769538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] AGENT.vxlan_udp_port = 4789 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.769967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] service_providers.service_provider = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.770388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] nova.auth_section = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.770821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] nova.auth_type = password {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.771250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] nova.cafile = /opt/stack/data/ca-bundle.pem {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.771680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] nova.certfile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.772186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] nova.collect_timing = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.772706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] nova.endpoint_type = public {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.773141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] nova.insecure = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.773547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] nova.keyfile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.773974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] nova.region_name = RegionOne {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.774391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] nova.split_loggers = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.774854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] nova.timeout = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.775272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] placement.auth_section = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.775772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] placement.auth_type = password {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.776197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] placement.cafile = /opt/stack/data/ca-bundle.pem {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.776604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] placement.certfile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.777023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] placement.collect_timing = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.777457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] placement.endpoint_type = public {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.777879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] placement.insecure = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.778286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] placement.keyfile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.778690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] placement.region_name = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.779102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] placement.split_loggers = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.779538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] placement.timeout = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.779965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.auth_section = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.780375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.auth_type = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.780779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.cafile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.781190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.certfile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.781599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.collect_timing = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.782214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.connect_retries = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.782645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.connect_retry_delay = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.783158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.enable_notifications = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.783585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.endpoint_override = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.783994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.insecure = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.784406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.interface = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.784810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.keyfile = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.785218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.max_version = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.785625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.min_version = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.786036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.region_name = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.786441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.service_name = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.786847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.service_type = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.787268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.split_loggers = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.787707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.status_code_retries = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.788112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.status_code_retry_delay = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.788515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.timeout = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.788926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.valid_interfaces = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.789333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ironic.version = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.789766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.bridge_mac_table_size = 50000 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.790191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.bridge_mappings = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.790620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.datapath_type = system {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.791050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.igmp_snooping_enable = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.791497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.int_peer_patch_port = patch-tun {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.791947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.integration_bridge = br-int {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.792446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.local_ip = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.792946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.of_connect_timeout = 300 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.793361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.of_inactivity_probe = 10 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.793800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.of_listen_address = 127.0.0.1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.794216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.of_listen_port = 6633 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.794628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.of_request_timeout = 300 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.795038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.openflow_processed_per_port = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.795504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.ovsdb_connection = tcp:127.0.0.1:6640 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.795922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.ovsdb_debug = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.796338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.ovsdb_timeout = 10 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.796769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.resource_provider_bandwidths = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.797179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.resource_provider_default_hypervisor = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.798201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.resource_provider_hypervisors = {} {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.798783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.resource_provider_inventory_defaults = {'allocation_ratio': 1.0, 'min_unit': 1, 'step_size': 1, 'reserved': 0} {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.799254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.resource_provider_packet_processing_inventory_defaults = {'allocation_ratio': 1.0, 'min_unit': 1, 'step_size': 1, 'reserved': 0} {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.799710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.resource_provider_packet_processing_with_direction = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.800264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.resource_provider_packet_processing_without_direction = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.800675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.ssl_ca_cert_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.801079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.ssl_cert_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.801492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.ssl_key_file = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.801931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.tun_peer_patch_port = patch-int {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.802454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.tunnel_bridge = br-tun {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.802881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] OVS.vhostuser_socket_dir = /var/run/openvswitch {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.803428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.amqp_auto_delete = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.803844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.amqp_durable_queues = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.804302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.conn_pool_min_size = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.804716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.conn_pool_ttl = 1200 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.805128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.direct_mandatory_flag = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.805536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.enable_cancel_on_failover = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.805946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.heartbeat_in_pthread = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.806361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.heartbeat_rate = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.806782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.heartbeat_timeout_threshold = 60 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.807189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.kombu_compression = None {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.807696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.kombu_failover_strategy = round-robin {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.808118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout = 60 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.808540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.kombu_reconnect_delay = 1.0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.808957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.rabbit_ha_queues = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.809369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.rabbit_interval_max = 30 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.809804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.rabbit_login_method = AMQPLAIN {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.810221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.rabbit_qos_prefetch_count = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.810631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.rabbit_quorum_queue = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.811049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.rabbit_retry_backoff = 2 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.811497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.rabbit_retry_interval = 1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.811908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.rabbit_transient_queues_ttl = 1800 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.812325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.rpc_conn_pool_size = 30 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.812834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.ssl = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.814155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.ssl_ca_file = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.814577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.ssl_cert_file = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.814986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.ssl_enforce_fips_mode = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.815450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.ssl_key_file = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.815886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_rabbit.ssl_version = {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.816319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_notifications.driver = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.816731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_notifications.retry = -1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.817192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_notifications.topics = ['notifications'] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.817605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_messaging_notifications.transport_url = **** {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.818032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] agent.extensions = [] {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.818446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] oslo_versionedobjects.fatal_exception_format_errors = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.818866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] QUOTAS.default_quota = -1 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.819292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] QUOTAS.quota_driver = neutron.db.quota.driver_nolock.DbQuotaNoLockDriver {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.819739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] QUOTAS.quota_network = 100 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.820166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] QUOTAS.quota_port = 500 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.820579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] QUOTAS.quota_rbac_policy = 10 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.821003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] QUOTAS.quota_security_group = 100 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.821440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] QUOTAS.quota_security_group_rule = 100 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.821864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] QUOTAS.quota_subnet = 100 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.822283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] QUOTAS.track_quota_usage = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.822791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] DHCP.dhcp_rebinding_time = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.823203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] DHCP.dhcp_renewal_time = 0 {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.823726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] DHCP.enable_ipv6 = True {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.824715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] LOCAL_IP.static_nat = False {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Jul 01 09:18:14.825077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.service [None req-472a93c9-85d9-4d3e-83d5-198b012b3264 None None] ******************************************************************************** {{(pid=106231) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2613}} Jul 01 09:18:14.836607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.rpc [-] Creating Consumer connection for Service l3_agent {{(pid=106231) start /usr/local/lib/python3.8/dist-packages/neutron_lib/rpc.py:311}} Jul 01 09:18:14.876204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] L3 agent started Jul 01 09:18:14.892636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:18:14.893033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Starting fullsync periodic_sync_routers_task {{(pid=106231) periodic_sync_routers_task /opt/stack/neutron/neutron/agent/l3/agent.py:875}} Jul 01 09:18:14.894371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running privsep helper: ['sudo', '/usr/local/bin/neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'privsep-helper', '--config-file', '/etc/neutron/neutron.conf', '--config-file', '/etc/neutron/l3_agent.ini', '--privsep_context', 'neutron.privileged.namespace_cmd', '--privsep_sock_path', '/tmp/tmpqzhpsc0i/privsep.sock'] Jul 01 09:18:14.895106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Starting _process_routers_loop {{(pid=106231) _process_routers_loop /opt/stack/neutron/neutron/agent/l3/agent.py:864}} Jul 01 09:18:14.918475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.pd [-] SIGUSR1 signal handler set {{(pid=106231) after_start /opt/stack/neutron/neutron/agent/linux/pd.py:330}} Jul 01 09:18:14.925783 dv-neutron-848393-1-hyperv sudo[107085]: ubuntu : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/usr/local/bin/neutron-rootwrap /etc/neutron/rootwrap.conf privsep-helper --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/l3_agent.ini --privsep_context neutron.privileged.namespace_cmd --privsep_sock_path /tmp/tmpqzhpsc0i/privsep.sock Jul 01 09:18:14.927320 dv-neutron-848393-1-hyperv sudo[107085]: pam_unix(sudo:session): session opened for user root by (uid=0) Jul 01 09:18:16.477127 dv-neutron-848393-1-hyperv sudo[107085]: pam_unix(sudo:session): session closed for user root Jul 01 09:18:16.485603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Spawned new privsep daemon via rootwrap Jul 01 09:18:16.487608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Accepted privsep connection to /tmp/tmpqzhpsc0i/privsep.sock {{(pid=106231) __init__ /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:362}} Jul 01 09:18:16.489411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] privsep daemon starting Jul 01 09:18:16.489863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] privsep process running with uid/gid: 0/0 Jul 01 09:18:16.490299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] privsep process running with capabilities (eff/prm/inh): CAP_SYS_ADMIN/CAP_SYS_ADMIN/none Jul 01 09:18:16.490732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] privsep daemon running as pid 107097 Jul 01 09:18:16.493171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67a197cd-fef0-46d8-b9d2-efc8aca62b7f]: (2,) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:17.139220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "context-manager" acquired by "neutron_lib.db.api._create_context_manager" :: waited 0.000s {{(pid=107097) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:18:17.139938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "context-manager" "released" by "neutron_lib.db.api._create_context_manager" :: held 0.000s {{(pid=107097) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:18:17.406306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2af8e324-cbca-46c2-a5b9-f0b318a4ac1c]: (4, []) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:17.407294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Time-cost: call 6a8ceefd-4b93-47c8-9e81-2f4f4c492815 function get_router_ids start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:18:17.545696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Time-cost: call 6a8ceefd-4b93-47c8-9e81-2f4f4c492815 function get_router_ids took 0.138s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:18:17.547110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] periodic_sync_routers_task successfully completed {{(pid=106231) fetch_and_sync_all_routers /opt/stack/neutron/neutron/agent/l3/agent.py:948}} Jul 01 09:18:45.810867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b94af53a-573f-4233-be0e-14272f46c6da demo admin] Got routers updated notification :['0a5215a0-df88-4c5c-a926-a3c7b3bf5f79'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:18:45.823393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 1, update_id 05a61c98-a13f-494d-8349-bc21d64974ea. Wait time elapsed: 0.001 Jul 01 09:18:45.823393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 1, update_id 05a61c98-a13f-494d-8349-bc21d64974ea. Wait time elapsed: 0.002 Jul 01 09:18:45.823393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:18:48.144821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.331s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:18:48.147742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:18:48.150211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a5215a0-df88-4c5c-a926-a3c7b3bf5f79), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:18:48.152042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:18:48.154577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:18:48.155412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] Running privsep helper: ['sudo', '/usr/local/bin/neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'privsep-helper', '--config-file', '/etc/neutron/neutron.conf', '--config-file', '/etc/neutron/l3_agent.ini', '--privsep_context', 'neutron.privileged.default', '--privsep_sock_path', '/tmp/tmpmi9ic220/privsep.sock'] Jul 01 09:18:48.169254 dv-neutron-848393-1-hyperv sudo[107851]: ubuntu : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/usr/local/bin/neutron-rootwrap /etc/neutron/rootwrap.conf privsep-helper --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/l3_agent.ini --privsep_context neutron.privileged.default --privsep_sock_path /tmp/tmpmi9ic220/privsep.sock Jul 01 09:18:48.170167 dv-neutron-848393-1-hyperv sudo[107851]: pam_unix(sudo:session): session opened for user root by (uid=0) Jul 01 09:18:49.493864 dv-neutron-848393-1-hyperv sudo[107851]: pam_unix(sudo:session): session closed for user root Jul 01 09:18:49.506153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] Spawned new privsep daemon via rootwrap Jul 01 09:18:49.506740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] Accepted privsep connection to /tmp/tmpmi9ic220/privsep.sock {{(pid=106231) __init__ /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:362}} Jul 01 09:18:49.507758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] privsep daemon starting Jul 01 09:18:49.507999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] privsep process running with uid/gid: 0/0 Jul 01 09:18:49.508247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] privsep process running with capabilities (eff/prm/inh): CAP_DAC_OVERRIDE|CAP_DAC_READ_SEARCH|CAP_NET_ADMIN|CAP_SYS_ADMIN|CAP_SYS_PTRACE/CAP_DAC_OVERRIDE|CAP_DAC_READ_SEARCH|CAP_NET_ADMIN|CAP_SYS_ADMIN|CAP_SYS_PTRACE/none Jul 01 09:18:49.508476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] privsep daemon running as pid 107952 Jul 01 09:18:49.509966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0523d79c-9589-4c6a-92fa-321c99fbe19a]: (2,) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:50.130163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "context-manager" acquired by "neutron_lib.db.api._create_context_manager" :: waited 0.000s {{(pid=107952) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:18:50.130832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "context-manager" "released" by "neutron_lib.db.api._create_context_manager" :: held 0.000s {{(pid=107952) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:18:50.377113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cb6c65b-b64d-4635-92f0-f17be67fcbee]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:50.404773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c180ff78-6b8c-4b1a-a2ad-97d5f1b1cfbf]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:50.425188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f667579e-6f28-4f91-969c-aa7f339f797f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:50.426420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] Running privsep helper: ['sudo', '/usr/local/bin/neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'privsep-helper', '--config-file', '/etc/neutron/neutron.conf', '--config-file', '/etc/neutron/l3_agent.ini', '--privsep_context', 'neutron.privileged.link_cmd', '--privsep_sock_path', '/tmp/tmp5w72ryrh/privsep.sock'] Jul 01 09:18:50.439254 dv-neutron-848393-1-hyperv sudo[107974]: ubuntu : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/usr/local/bin/neutron-rootwrap /etc/neutron/rootwrap.conf privsep-helper --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/l3_agent.ini --privsep_context neutron.privileged.link_cmd --privsep_sock_path /tmp/tmp5w72ryrh/privsep.sock Jul 01 09:18:50.440176 dv-neutron-848393-1-hyperv sudo[107974]: pam_unix(sudo:session): session opened for user root by (uid=0) Jul 01 09:18:51.749870 dv-neutron-848393-1-hyperv sudo[107974]: pam_unix(sudo:session): session closed for user root Jul 01 09:18:51.756197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] Spawned new privsep daemon via rootwrap Jul 01 09:18:51.757767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] Accepted privsep connection to /tmp/tmp5w72ryrh/privsep.sock {{(pid=106231) __init__ /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:362}} Jul 01 09:18:51.758908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] privsep daemon starting Jul 01 09:18:51.759040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] privsep process running with uid/gid: 0/0 Jul 01 09:18:51.761552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] privsep process running with capabilities (eff/prm/inh): CAP_NET_ADMIN|CAP_SYS_ADMIN/CAP_NET_ADMIN|CAP_SYS_ADMIN/none Jul 01 09:18:51.761804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo.privsep.daemon [-] privsep daemon running as pid 107978 Jul 01 09:18:51.764220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71c8990e-1a08-4959-96a1-135f79fc297e]: (2,) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:52.376464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "context-manager" acquired by "neutron_lib.db.api._create_context_manager" :: waited 0.000s {{(pid=107978) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:18:52.376749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "context-manager" "released" by "neutron_lib.db.api._create_context_manager" :: held 0.000s {{(pid=107978) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:18:52.661384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ec4a043-f843-4c46-b1eb-b11304746029]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107998, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:52.677903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f22cd4d-5af1-4781-864c-a6c216be6851]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:52.696980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[044aa161-f911-4c33-9542-5e46b604c07c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:52.717087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4224ca4-30b5-4eac-acc8-176588ea52e7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:52.736777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e192d623-92c8-4531-82ba-d6c8efd953f9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:52.737743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:18:52.738052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:18:52.738378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:18:52.738711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fa0ff57-49b7-4c01-8fdc-8855d809fdc0) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:18:53.053975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:18:53.055120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c8dfabd-c949-485c-b47d-c83512ceeb5f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:53.058698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb138909-4b2a-4235-b3d9-0c1d5e7da8c2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:53.135040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl [-] Created lookup_table index Flow_Table.name {{(pid=106231) autocreate_indices /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/__init__.py:85}} Jul 01 09:18:53.135295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl [-] Created lookup_table index Mirror.name {{(pid=106231) autocreate_indices /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/__init__.py:85}} Jul 01 09:18:53.135603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl [-] Created schema index Bridge.name {{(pid=106231) autocreate_indices /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/__init__.py:104}} Jul 01 09:18:53.135882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl [-] Created schema index Port.name {{(pid=106231) autocreate_indices /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/__init__.py:104}} Jul 01 09:18:53.136141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl [-] Created schema index Interface.name {{(pid=106231) autocreate_indices /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/__init__.py:104}} Jul 01 09:18:53.136388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl [-] Created schema index Manager.target {{(pid=106231) autocreate_indices /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/__init__.py:104}} Jul 01 09:18:53.218786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fa0ff57-49, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:53.219057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:18:53.219849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fa0ff57-49, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:53.220512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:53.221277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:53.221664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:53.222041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fa0ff57-49, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fa0ff57-49b7-4c01-8fdc-8855d809fdc0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0f:4c:08'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:53.250605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06680145-7f3c-4fb5-91a7-24445fbc03d1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:53.282201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[240e0efe-da9a-4142-86f7-3b35b35ac89a]: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:18:53.282201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:18:53.282201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:18:53.282201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:18:53.282201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:18:53.282201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:18:53.282201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:18:53.282201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:18:53.282201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:18:53.282201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:18:53.282201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:18:53.282201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:18:53.282201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:53.283192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[240e0efe-da9a-4142-86f7-3b35b35ac89a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:18:53.285541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:18:53.286603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:18:53.286603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:18:53.286603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:18:53.286603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:18:53.286603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:18:53.286603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:18:53.286603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:18:53.286603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:18:53.286603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:18:53.286603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:53.286603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:18:53.287341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:53.288581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:18:53.289688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d09c673-fdbb-4642-9888-ee8d832cabd0]: (4, ['qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:53.289688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:18:53.289688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:18:53.289911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:18:53.290116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:18:53.291166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8183fc49-9474-47c1-a154-f6c1e9c1975f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:53.292049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ee7cbd8-c3f8-4477-a46a-51f7be896b55]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:53.292893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[237f2feb-1a88-4a5f-8766-479fc65934c2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:53.313024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e43cd564-8771-49f5-8295-048b552a3e32]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 202892, 'reachable_time': 21896, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 108033, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:53.327963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:18:53.328796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5875b2a-ab89-4d3f-893f-4ec7ed97cf6d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:53.329876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "router1", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:34Z", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:45Z", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 2, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fa0ff57-49b7-4c01-8fdc-8855d809fdc0", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b71076a-940c-4868-a739-6596faed3a10", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0f:4c:08", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 27, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 26 Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:44Z", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:47Z", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/26", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": "c8ea00fe-9c19-4351-88b9-0cff9ceac3e0" Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id 05a61c98-a13f-494d-8349-bc21d64974ea. Wait time elapsed: 7.520 Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id 05a61c98-a13f-494d-8349-bc21d64974ea. Wait time elapsed: 7.521 Jul 01 09:18:53.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:18:53.571130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-623019f3-1a5a-46f3-8dc0-b891386efbdd demo admin] Got routers updated notification :['0a5215a0-df88-4c5c-a926-a3c7b3bf5f79'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:18:54.889997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:18:56.302177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.970s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:18:56.303164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:18:56.304634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a5215a0-df88-4c5c-a926-a3c7b3bf5f79), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:18:56.304968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:18:56.305260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:18:56.306518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83c3848a-7dd9-448f-9e86-f58730df9104]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.321048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4da51c5b-7aa0-4557-b060-691e641678d2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.332131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b40bb9f-890a-4d79-babd-adf73d4b0532]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.367917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4e9eae7-c9cb-46ee-8cc7-60f259f49b92]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 108295, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.387926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79bdedbd-6a58-4bc0-9173-4267d4737709]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.407044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05800617-4d3e-4daf-99c3-70be3680a481]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.427700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f691124b-e0b2-475e-8c70-8a44c6c6089d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.438530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c93da1df-803d-4e46-bfd4-49053f1d3ddb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.439340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:18:56.441072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:18:56.441325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:18:56.441603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fa0ff57-49b7-4c01-8fdc-8855d809fdc0) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:18:56.457002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:18:56.457988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd98c731-f3e6-476d-9840-e710045aecfa]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.460437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13dde4d6-2332-4c9a-90fe-31633ba0a053]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.461656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fa0ff57-49, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:56.495408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fa0ff57-49, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:56.497614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:56.498015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:56.498407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:56.498790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fa0ff57-49, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fa0ff57-49b7-4c01-8fdc-8855d809fdc0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0f:4c:08'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:56.530187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be856c9a-a43f-48a0-9f56-ccdcd37f255d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.560331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f707293c-14ee-48ce-8a8e-a9f68b48edc0]: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:18:56.560331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:18:56.560331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:18:56.560331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:18:56.560331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:18:56.560331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:18:56.560331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:18:56.560331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:18:56.560331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:18:56.560331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:18:56.560331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:18:56.560331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:18:56.560331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:56.561214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f707293c-14ee-48ce-8a8e-a9f68b48edc0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:56.561565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:56.563322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:18:56.565201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[457f2e15-d246-44d0-b394-9e4c889d0b8f]: (4, ['qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.565201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:18:56.565451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:18:56.565695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:18:56.566037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:18:56.567063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09b47658-4103-46e3-b586-a8d64836e8ee]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.568488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d0dab4e-59f0-4938-9125-1702b70ffe65]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.569717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bccdad4f-b6c4-4289-832e-c5d4f6900263]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.588732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac5d3099-4c71-4696-a0d8-d92612b58afc]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 203262, 'reachable_time': 44996, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 108435, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.593985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:18:56.594246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ae2864b-9d3a-4e4c-9034-d319bd3a7419]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:56.595286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "router1", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43" Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:34Z", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:52Z", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:50:2e", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 34, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:52Z", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:55Z", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fa0ff57-49b7-4c01-8fdc-8855d809fdc0", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b71076a-940c-4868-a739-6596faed3a10", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0f:4c:08", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 27, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 26 Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:44Z", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:54Z", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/26", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": "c8ea00fe-9c19-4351-88b9-0cff9ceac3e0" Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:18:56.597684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 1, update_id cbe84ffb-c3d7-467b-807d-482c5cb0c868. Wait time elapsed: 3.025 Jul 01 09:18:56.610279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 1, update_id cbe84ffb-c3d7-467b-807d-482c5cb0c868. Wait time elapsed: 3.025 Jul 01 09:18:56.610279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:18:57.460482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.863s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:18:57.461039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:18:57.461626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a5215a0-df88-4c5c-a926-a3c7b3bf5f79), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:18:57.462112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:18:57.462579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:18:57.463761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57ad98a1-b4a1-49b0-823b-5557dcc23863]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.485846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53f11085-eb22-4d5a-ba72-a3753f727280]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.496121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6366ecfd-5401-450c-b0c1-7c008f9cac94]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.548979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ef9c565-0f19-4a9c-a8ac-9d7b8ed1b03e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 108607, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.565476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5878aac4-1576-41d8-9e7a-f7df4444132a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.584185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cdf7abaf-0d52-4567-958e-1d6ac2dca5bd]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.605742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5b30aa3-c3ad-4b09-9bb8-67a9cab5bf55]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.630991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c89854c-bf62-4c0f-9490-199c2fe22aff]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.631592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:18:57.631899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:18:57.632169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:18:57.632468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fa0ff57-49b7-4c01-8fdc-8855d809fdc0) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:18:57.646706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:18:57.647198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7aa0dbf0-5220-41fa-abee-48aca21fc821]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.649899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee86b476-c502-47f0-a308-e6612adfb713]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.650874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fa0ff57-49, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:57.675038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fa0ff57-49, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:57.675646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:57.676030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:57.676570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:57.677093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fa0ff57-49, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fa0ff57-49b7-4c01-8fdc-8855d809fdc0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0f:4c:08'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:18:57.710442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ac6a747-32a5-4560-9075-43f8c8bbda53]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.743742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e6371aa0-c4cb-429f-98b2-365eae4238c0]: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:18:57.743742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:18:57.743742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:18:57.743742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:18:57.743742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:18:57.743742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:18:57.743742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:18:57.743742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:18:57.743742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:18:57.743742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:18:57.743742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:18:57.743742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:18:57.743742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:57.744667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6371aa0-c4cb-429f-98b2-365eae4238c0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:57.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:57.746573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:18:57.749897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d9826e2-8aa2-4e8b-b6ff-f3f7a698bbe8]: (4, ['qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.750661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:18:57.751065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:18:57.751520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:18:57.752027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:18:57.753298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84e032ea-2294-4130-93ac-544804e5a0b5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.754398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3663fc93-fbf1-4af2-8646-782ab5acc22a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.755759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6a471a7-a894-49b6-91b0-4a401b65fb77]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.778736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f147ee35-cb10-4666-8f92-6d815c1b243b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 203380, 'reachable_time': 34044, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 108648, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.784131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:18:57.784591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8932d35e-4341-4d62-801a-e3b2b2f57f1d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:18:57.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "router1", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43" Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:34Z", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:52Z", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:50:2e", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 34, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:52Z", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:55Z", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fa0ff57-49b7-4c01-8fdc-8855d809fdc0", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b71076a-940c-4868-a739-6596faed3a10", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0f:4c:08", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 27, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 26 Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:44Z", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:54Z", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/26", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": "c8ea00fe-9c19-4351-88b9-0cff9ceac3e0" Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:18:57.789890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:18:57.816176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id 05a61c98-a13f-494d-8349-bc21d64974ea. Wait time elapsed: 11.998 Jul 01 09:18:57.816176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id 05a61c98-a13f-494d-8349-bc21d64974ea. Wait time elapsed: 11.998 Jul 01 09:18:57.816176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:18:59.774170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.964s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:18:59.774746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:18:59.775792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a5215a0-df88-4c5c-a926-a3c7b3bf5f79), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:18:59.776208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:18:59.776601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:18:59.778358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47f4ab56-9457-46f1-913b-d282a1e6e455]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:59.797580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a44cf008-e747-4652-b314-7029dd23cdbc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:59.844691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fbc81b4-3428-4de2-8591-a98ef7105f8a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:59.882796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4804c693-257c-45fa-8157-96fd39bede18]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 109170, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:59.904144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[677be7fe-5fb0-464f-8210-e340dd8f2909]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:59.926009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[023da1b3-e586-445d-8138-e2db36bf656f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:18:59.971052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82491b20-d821-4aaa-8524-5c576fa18cb2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:00.000659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b960894-f2ad-4077-bf55-fcff36a7bfee]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:00.001531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:00.002593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:00.003113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:19:00.003623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fa0ff57-49b7-4c01-8fdc-8855d809fdc0) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:19:00.020542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:19:00.021411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d341898-e1b2-41d7-a16b-dcb56f0e0cde]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:00.025103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6daf719-a746-45c8-b35d-d34c5e030b9e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:00.026340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fa0ff57-49, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:00.073284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fa0ff57-49, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:00.074266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:00.075241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:00.075793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:00.076266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fa0ff57-49, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fa0ff57-49b7-4c01-8fdc-8855d809fdc0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0f:4c:08'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:00.106976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9825d640-22cf-411e-b4c5-5fa07340c45d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:00.168790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b2ac8c6d-0632-427a-859b-4eefc70eb2fb]: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:19:00.168790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:19:00.168790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:19:00.168790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:19:00.168790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:19:00.168790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:19:00.168790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:19:00.168790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:19:00.168790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:19:00.168790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:19:00.168790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:19:00.168790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:19:00.168790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:00.169754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2ac8c6d-0632-427a-859b-4eefc70eb2fb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:00.169997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:00.171878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:00.173981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3db9d5c-c781-4d31-81e5-4f407428fa29]: (4, ['qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:00.173981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:00.173981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:00.174222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:19:00.174401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:00.175596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[565b4595-1ed3-4f35-9a55-f67ba497dfe8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:00.176647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[765546c5-c5ae-4dde-ac4e-077aafe997ec]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:00.177922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5222f89b-747a-43e0-9984-eb207667939d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:00.218755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb72c5e2-ceb2-443e-9342-334595047d43]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 203614, 'reachable_time': 42900, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 109250, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:00.224398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:19:00.224813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5a15e55-f6e0-459a-96db-c27775b37e4e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:00.225967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "router1", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43" Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:34Z", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:52Z", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:50:2e", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 34, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:52Z", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:55Z", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fa0ff57-49b7-4c01-8fdc-8855d809fdc0", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b71076a-940c-4868-a739-6596faed3a10", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0f:4c:08", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 27, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 26 Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:44Z", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:58Z", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/26", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": "c8ea00fe-9c19-4351-88b9-0cff9ceac3e0" Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:19:00.229551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:19:00.242205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id cbe84ffb-c3d7-467b-807d-482c5cb0c868. Wait time elapsed: 6.666 Jul 01 09:19:00.242205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id cbe84ffb-c3d7-467b-807d-482c5cb0c868. Wait time elapsed: 6.666 Jul 01 09:19:00.242205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:19:01.364506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.126s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:19:01.365726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:19:01.367669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a5215a0-df88-4c5c-a926-a3c7b3bf5f79), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:19:01.367669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:01.367669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:01.368593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1bbafa70-f4f5-43bd-bb7d-6270b4faaf7e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.383680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d222ceb-3f29-4870-a2ef-e360e9ccec28]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.421497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df5643f4-084b-4cc8-99d3-265c5aa9b7ff]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.471770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3cfd5f3a-f7d0-4ffb-9f94-4d9ea0702f37]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 109718, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.498085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44639b39-7fbc-40af-8439-52ecca75548d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.538291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[444f8eb6-5bbe-43ec-86bd-0e8f638747fe]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.567915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f24a5127-72a9-4fbb-b062-29667be3cede]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.612977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67de2a0a-db36-4d07-abfd-bfd92b76bb4f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.614288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:01.614814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:01.615232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:19:01.615684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fa0ff57-49b7-4c01-8fdc-8855d809fdc0) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:19:01.649108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:19:01.649615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7dcb95da-2919-4088-891d-cb40c0791308]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.652306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[936371bb-53c8-4f16-9e66-2be1314d4a2a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.653786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fa0ff57-49, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:01.685022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fa0ff57-49, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:01.685888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:01.687276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:01.691886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:01.692445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fa0ff57-49, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fa0ff57-49b7-4c01-8fdc-8855d809fdc0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0f:4c:08'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:01.775820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbdcf1bc-b793-4c0f-8f00-6c82ed78cf14]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.837064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ffd73697-4822-47ef-9a08-8301b1dec2eb]: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:19:01.837064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:19:01.837064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:19:01.837064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:19:01.837064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:19:01.837064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:19:01.837064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:19:01.837064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:19:01.837064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:19:01.837064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:19:01.837064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:19:01.837064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:19:01.837064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:01.838364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffd73697-4822-47ef-9a08-8301b1dec2eb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:01.841083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:01.843571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:01.847395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72c63516-0793-4d6b-bea7-b96c56bd12bb]: (4, ['qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.848247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:01.848729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:01.849132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:19:01.849618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:01.850751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b2a5979-2bbd-4c42-b751-2d5c0a7df22d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.852351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a2910ae-9eef-40fe-af3b-656f2184e921]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.857102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf590fd0-e514-43a8-afd9-d38f68c739e5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.902603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1116b7b-54a2-4b7a-b37e-5f3207381e4e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 203773, 'reachable_time': 35748, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 109873, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.908462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:19:01.908462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d453f4f-ea1c-4005-b518-ad1853a948f6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:01.909235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "router1", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43" Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:34Z", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:52Z", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:50:2e", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 34, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:52Z", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:55Z", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fa0ff57-49b7-4c01-8fdc-8855d809fdc0", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b71076a-940c-4868-a739-6596faed3a10", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0f:4c:08", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 27, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 26 Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:44Z", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:59Z", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/26", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": "c8ea00fe-9c19-4351-88b9-0cff9ceac3e0" Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:19:01.911786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id 05a61c98-a13f-494d-8349-bc21d64974ea. Wait time elapsed: 16.099 Jul 01 09:19:01.929313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id 05a61c98-a13f-494d-8349-bc21d64974ea. Wait time elapsed: 16.099 Jul 01 09:19:01.929313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:19:05.936957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.026s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:19:05.937811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:19:05.938625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a5215a0-df88-4c5c-a926-a3c7b3bf5f79), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:19:05.938961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:05.939248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:05.940521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72d77bbc-81d8-4d1a-9878-00d67fd294dc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:05.997339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc7acca1-5a39-41db-a870-ac7271f432bd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.023118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6fed14d1-3a7e-49b7-b0cd-e5dce566fd18]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.087187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4214daa-a86b-499a-9d9b-b3d9d1b0bf72]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 110586, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.128141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0763249-dcd1-43bd-b181-f3b03698affe]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.158138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7f94ff4-5093-4f4a-abde-75abdd22c52f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.184565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76b20b86-6ee9-4390-bd02-f12dd7852976]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.223947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9183d86-e9ea-41e0-9344-8f72ba0bec7f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.224548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:06.224828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:06.225095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:19:06.225375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fa0ff57-49b7-4c01-8fdc-8855d809fdc0) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:19:06.259958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:19:06.260409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a853ed57-dd82-48ec-af73-a2912aa5c8f5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.263953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c0b938f-4a6a-4898-bd7a-b96384ec5f7e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.265637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fa0ff57-49, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:06.296486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fa0ff57-49, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:06.297145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:06.298350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:06.299820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:06.299820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fa0ff57-49, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fa0ff57-49b7-4c01-8fdc-8855d809fdc0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0f:4c:08'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:06.358124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25f03dbf-cbc7-4f00-93dc-c7d4610e932f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.399987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[14f0d360-4f28-453e-bf14-93a0291c9265]: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:19:06.399987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:19:06.399987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:19:06.399987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:19:06.399987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:19:06.399987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:19:06.399987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:19:06.399987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:19:06.399987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:19:06.399987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:19:06.399987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:19:06.399987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:19:06.399987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:06.400818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14f0d360-4f28-453e-bf14-93a0291c9265]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:06.401976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:06.405791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:06.407783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8ba53fb-b07b-467a-9784-7d58e98f4a01]: (4, ['qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.407783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:06.408086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:06.408495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:19:06.408975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:06.410461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb273915-82dc-4c6f-8a65-ee0096d529a5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.411391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b59a405e-6d2e-4dc0-8cdd-d18119238872]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.412380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a3c2f99-4f1c-4307-8216-bab351dba018]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.457558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05d2d7eb-7f53-451a-a710-e2e1b73632d3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 204234, 'reachable_time': 29672, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 110759, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.464330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:19:06.464330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6091154-7793-4cdd-9400-ed73dad768da]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:06.465728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "router1", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43" Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:34Z", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:52Z", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:50:2e", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 34, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:52Z", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:55Z", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fa0ff57-49b7-4c01-8fdc-8855d809fdc0", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b71076a-940c-4868-a739-6596faed3a10", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0f:4c:08", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 27, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 26 Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:44Z", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:19:03Z", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/26", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": "c8ea00fe-9c19-4351-88b9-0cff9ceac3e0" Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:19:06.471230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:19:06.483032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id cbe84ffb-c3d7-467b-807d-482c5cb0c868. Wait time elapsed: 12.906 Jul 01 09:19:06.483032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id cbe84ffb-c3d7-467b-807d-482c5cb0c868. Wait time elapsed: 12.907 Jul 01 09:19:06.484123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:19:07.926347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.448s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:19:07.926991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:19:07.927637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a5215a0-df88-4c5c-a926-a3c7b3bf5f79), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:19:07.927980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:07.928266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:07.929632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c859de3-1b24-4473-9ba1-1e82a79dc458]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:07.963649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b26abd4e-af0f-4a93-95e3-af969d2de12c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:07.998696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f8d7202-21f5-4a3a-bda1-c91edc749567]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.058194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[170bd8e5-7415-4c9d-94ae-38f2f2b3e1fd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 110972, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.093194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[018ac842-067d-47d7-adc4-824f13d82816]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.127108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23d29c00-2dc6-4a95-a507-bc03979c9503]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.168711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad0a03b8-0a60-42bc-95f1-b5f6d0ac44d1]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.186623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abcee6d0-f9ff-4318-bf57-96451c834d57]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.188103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:08.188566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:08.189687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:19:08.189687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fa0ff57-49b7-4c01-8fdc-8855d809fdc0) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:19:08.225143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:19:08.225433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11bc47e5-121b-41c6-a4d4-b3e89dc691c3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.265973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42571b4e-90cb-40b6-8ce3-763d9c37b6a5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.268515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fa0ff57-49, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:08.310420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fa0ff57-49, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:08.311237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:08.311737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:08.312530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:08.313001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fa0ff57-49, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fa0ff57-49b7-4c01-8fdc-8855d809fdc0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0f:4c:08'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:08.379394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13ac6774-2f46-46af-85cd-7b195344efea]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.422854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cb5a4a80-b512-4c5d-b8fc-0470e34488e5]: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:19:08.422854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:19:08.422854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:19:08.422854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:19:08.422854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:19:08.422854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:19:08.422854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:19:08.422854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:19:08.422854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:19:08.422854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:19:08.422854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:19:08.422854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:19:08.422854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:08.423917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb5a4a80-b512-4c5d-b8fc-0470e34488e5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:08.424101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:08.425518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:08.427346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96510dca-519b-487a-914c-d09376a287a4]: (4, ['qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.427346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:08.427346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:08.427346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:19:08.427632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:08.428994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d02c2b9b-0e96-4174-bbdd-b44fc03b055f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.429984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed5c1a17-d4c4-4104-997b-f378934c96fe]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.431163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee8efb93-d029-4c9c-925a-8da57e121d0d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.451652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc164b68-ce99-4289-8869-99cb4e819401]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 204432, 'reachable_time': 27620, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 111007, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.460019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:19:08.460696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aebc9583-f81d-4d1c-b384-b862d4e196e8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:08.462372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "router1", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43" Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:34Z", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:52Z", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:50:2e", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 34, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:52Z", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:55Z", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fa0ff57-49b7-4c01-8fdc-8855d809fdc0", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b71076a-940c-4868-a739-6596faed3a10", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0f:4c:08", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 27, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 26 Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:44Z", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:19:03Z", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/26", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": "c8ea00fe-9c19-4351-88b9-0cff9ceac3e0" Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:19:08.470245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:19:08.504038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id 05a61c98-a13f-494d-8349-bc21d64974ea. Wait time elapsed: 22.674 Jul 01 09:19:08.504038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id 05a61c98-a13f-494d-8349-bc21d64974ea. Wait time elapsed: 22.675 Jul 01 09:19:08.504038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:19:09.846160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.359s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:19:09.847091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:19:09.848475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a5215a0-df88-4c5c-a926-a3c7b3bf5f79), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:19:09.849022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:09.849862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:09.851424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6c12946-b91a-4dba-9665-a880b3dac9fa]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:09.872683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f6bf522-4b6a-496d-a808-8283b3c42852]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:09.897616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20c51344-d663-4c84-a405-c4c532b7428e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:09.945434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c57119e-de7f-4fbc-b2e2-aa9d9258eb84]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 111019, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:09.988733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51a3a267-1016-4499-bce6-f9a24f5cf91d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.016217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fda880ff-c673-4c96-b1d2-362d0a137f78]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.047245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b0f830e-8286-4599-9fa5-698244426ea6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.072766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3afe7d1f-17fc-47ee-a7da-e9b746be06ee]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.073665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:10.074848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:10.080341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:19:10.085350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fa0ff57-49b7-4c01-8fdc-8855d809fdc0) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:19:10.113188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:19:10.113909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4273cc6-2779-464f-b222-6cf7fd74dc9b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.119191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ed3aef4-a314-4519-8a14-9f28d1a19d1f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.121768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fa0ff57-49, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:10.169629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fa0ff57-49, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:10.175422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:10.177407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:10.178011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:10.179856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fa0ff57-49, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fa0ff57-49b7-4c01-8fdc-8855d809fdc0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0f:4c:08'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:10.226208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ba2b4b2-7c15-4533-bfd1-858a583512e4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.264447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3727e88e-5616-4b5d-9687-1cb1b77f6372]: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:19:10.264447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:19:10.264447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:19:10.264447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:19:10.264447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:19:10.264447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:19:10.264447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:19:10.264447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:19:10.264447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:19:10.264447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:19:10.264447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:19:10.264447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:19:10.264447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:10.266700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3727e88e-5616-4b5d-9687-1cb1b77f6372]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:10.268013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:10.270803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:10.272742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46c96492-1574-4226-85d8-010e11e5783f]: (4, ['qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.273515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:10.273620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:10.273883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:19:10.274233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:10.275212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0cc89118-b4dc-4322-b57d-ba80adfe6ab9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.276461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f0b0ce1-6ea6-46d0-abaf-b97a5dee53de]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.277565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34266f51-7482-4db8-9052-f060d3a1de67]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.311085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49919772-12f0-49fb-b836-39a67b76cc51]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 204620, 'reachable_time': 22596, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 111037, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.320327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:19:10.320757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6f50e22-805e-4409-9163-7e582cf1d7a5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:10.322018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "router1", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43" Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:34Z", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:52Z", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:50:2e", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 34, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:52Z", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:55Z", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fa0ff57-49b7-4c01-8fdc-8855d809fdc0", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b71076a-940c-4868-a739-6596faed3a10", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0f:4c:08", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 27, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 26 Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:44Z", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:19:08Z", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/26", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": "c8ea00fe-9c19-4351-88b9-0cff9ceac3e0" Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:19:10.324509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id cbe84ffb-c3d7-467b-807d-482c5cb0c868. Wait time elapsed: 16.752 Jul 01 09:19:10.345102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id cbe84ffb-c3d7-467b-807d-482c5cb0c868. Wait time elapsed: 16.752 Jul 01 09:19:10.345102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:19:11.464579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.141s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:19:11.465562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:19:11.466630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a5215a0-df88-4c5c-a926-a3c7b3bf5f79), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:19:11.467542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:11.468428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:11.470777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67ab0574-89f2-419d-b3f4-972dd8d2835e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.482165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c32f51d3-decf-43b8-90f0-3837aad0ff91]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.504704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a295de84-2c37-4547-b986-2fba764cb9c3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.541358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[115e7eb4-06ed-4f7d-9b83-319dc41aafb0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 111218, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.564569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[617148c9-9609-4da7-8321-f0fbaa8bc1db]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.587874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3c9e28f-74d2-4469-aca2-92b2e4090001]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.609300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09a7f901-8c7e-412a-9f93-85d6f10b43f0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.629746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e152cb35-0550-4a26-8807-484f4df5854a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.630988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:11.631546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:11.632091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:19:11.632662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fa0ff57-49b7-4c01-8fdc-8855d809fdc0) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:19:11.657220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:19:11.657793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95b372cd-afb6-432b-bba4-e6675899a269]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.660824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[932885db-3e5c-4d15-9708-409ff3b31443]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.661977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fa0ff57-49, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:11.695027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fa0ff57-49, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:11.695717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:11.699991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:11.701249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:11.701249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fa0ff57-49, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fa0ff57-49b7-4c01-8fdc-8855d809fdc0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0f:4c:08'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:11.756429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00e0ceb9-3a81-46f1-8962-89b53aece342]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.820035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2c30f325-f7a2-4e60-a16b-6366de9d4ef4]: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:19:11.820035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:19:11.820035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:19:11.820035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:19:11.820035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:19:11.820035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:19:11.820035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:19:11.820035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:19:11.820035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:19:11.820035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:19:11.820035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:19:11.820035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:19:11.820035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:11.821442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c30f325-f7a2-4e60-a16b-6366de9d4ef4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:11.822589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:11.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:11.827660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e6430c9-aef0-4959-aaff-e3aa11500bb0]: (4, ['qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.828712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:11.829278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:11.829898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:19:11.830449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:11.834101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb2267ba-f616-429c-847b-e3a588e117d9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.835092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47c9ed72-6d50-4a7d-8470-8909e3aaa1b0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.851355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f349436c-1c59-4551-a9df-7603ed7f3f29]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.899939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75bb5252-c09f-48db-aa03-26dc54646f76]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 204780, 'reachable_time': 19432, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 111361, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.902910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:19:11.903405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ca4b96a-0252-49e5-ad9e-00ec9e9e6ff3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:11.906833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "router1", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43" Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:34Z", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:52Z", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:50:2e", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 34, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:52Z", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:55Z", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fa0ff57-49b7-4c01-8fdc-8855d809fdc0", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b71076a-940c-4868-a739-6596faed3a10", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0f:4c:08", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 27, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 26 Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:44Z", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:19:10Z", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/26", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": "c8ea00fe-9c19-4351-88b9-0cff9ceac3e0" Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:19:11.913094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:19:11.933448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id 05a61c98-a13f-494d-8349-bc21d64974ea. Wait time elapsed: 26.111 Jul 01 09:19:11.933448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id 05a61c98-a13f-494d-8349-bc21d64974ea. Wait time elapsed: 26.112 Jul 01 09:19:11.933448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:19:14.140485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.212s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:19:14.141396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:19:14.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a5215a0-df88-4c5c-a926-a3c7b3bf5f79), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:19:14.142994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:14.143488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:14.145007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eba2d98b-4041-4063-92b6-6a352c99ee05]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.175026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abd4a1fd-c75f-48b9-b52c-292d7f505a89]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.208013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c34b7654-aff1-46ff-91b7-3c0d0575f126]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.223152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:14.223412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:14.256713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70bddc2f-69aa-49e8-83f9-2ec65529bd05]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 111735, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.294022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[424db4fd-6fcd-4ce2-82ae-69fdbd3de354]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.319041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7da2f44b-429f-4fc3-988d-bf72a766b22c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.361989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9df050b-eac6-48f8-a3aa-8c4eaac57332]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.398805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2227bb3a-5856-4616-a8a5-3dc23dc61b5e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.399791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:14.400068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:14.400339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:19:14.400637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fa0ff57-49b7-4c01-8fdc-8855d809fdc0) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:19:14.428547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:19:14.429107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97246cbd-dabd-4ed5-811b-0606208fc7ab]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.431798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da43a591-69fe-4ad0-8265-f5ff6886460a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.432991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fa0ff57-49, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:14.462715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fa0ff57-49, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:14.463707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:14.464272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:14.464803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:14.465321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fa0ff57-49, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fa0ff57-49b7-4c01-8fdc-8855d809fdc0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0f:4c:08'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:14.509713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:14.510038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:14.521753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[665e0b55-ac7f-44c7-9e8d-15f258769125]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.571468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5306c2b4-8b71-43a3-aa4f-c808f2ce25ce]: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:19:14.571468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:19:14.571468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:19:14.571468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:19:14.571468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:19:14.571468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:19:14.571468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:19:14.571468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:19:14.571468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:19:14.571468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:19:14.571468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:19:14.571468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:19:14.571468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:14.572637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5306c2b4-8b71-43a3-aa4f-c808f2ce25ce]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:14.573502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:14.575550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:14.579883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a12e0a6-23ee-421a-8bb0-a002fe9eb351]: (4, ['qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.580760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:14.581235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:14.581963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:19:14.582476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:14.583733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40de2d97-caf3-4298-bde8-e2c65373976b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.585001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1e8e0ad-8b0d-4aed-854b-9f7053d99439]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.588061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[283700c3-9a5c-4535-98c8-da9fb9022839]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.617614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6181463d-705b-4945-8a50-69baa545d5e4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 205052, 'reachable_time': 32108, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 111758, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.625798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:19:14.626789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a78ee298-085d-4b02-939d-7054221a891d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:14.628058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "router1", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43" Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:34Z", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:52Z", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:50:2e", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 34, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:52Z", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:55Z", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fa0ff57-49b7-4c01-8fdc-8855d809fdc0", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b71076a-940c-4868-a739-6596faed3a10", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0f:4c:08", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 27, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 26 Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:44Z", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:19:12Z", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/26", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": "c8ea00fe-9c19-4351-88b9-0cff9ceac3e0" Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:19:14.631944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:19:14.646257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3 Jul 01 09:19:14.646257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id cbe84ffb-c3d7-467b-807d-482c5cb0c868. Wait time elapsed: 21.072 Jul 01 09:19:14.646257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id cbe84ffb-c3d7-467b-807d-482c5cb0c868. Wait time elapsed: 21.072 Jul 01 09:19:14.646257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:19:15.926180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.281s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:19:15.927216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:19:15.928897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a5215a0-df88-4c5c-a926-a3c7b3bf5f79), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:19:15.929974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:15.930604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:15.933250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[740ca741-710c-4894-917b-130d42771e9d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:15.960927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca5c1a7f-d564-4487-a19e-e0f3da89ea48]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:15.997107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[814fde19-9bfc-4665-aacd-504c5e83eb2c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.044526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[940ec9fe-8289-4802-875f-fd7aa69fd9ed]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 111777, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.078066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f26904dd-14c3-4816-8353-15ca1e4ec808]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.110344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13514a00-4be7-4a10-9a71-44974c177459]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.137310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de8859a0-06ab-4b85-8369-79fcb55e368c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.170273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a137fac-80a1-4d64-90a1-1b8f7bc8a742]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.171094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:16.171551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:16.171956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:19:16.172374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fa0ff57-49b7-4c01-8fdc-8855d809fdc0) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:19:16.201899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:19:16.202166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9cf3c3e-74bd-401b-8f93-cfced715cea4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.205029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a9cffc6-837c-44bf-9782-2d1fd4a5c831]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.207600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fa0ff57-49, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:16.238699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fa0ff57-49, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:16.239560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:16.240118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:16.240647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:16.241170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fa0ff57-49, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fa0ff57-49b7-4c01-8fdc-8855d809fdc0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0f:4c:08'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:16.271014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d4033c0-a586-41e3-acc3-0d340293bdaf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.341236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[105b7575-594d-4442-8561-f8a7b21abdc1]: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:19:16.341236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:19:16.341236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:19:16.341236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:19:16.341236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:19:16.341236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:19:16.341236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:19:16.341236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:19:16.341236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:19:16.341236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:19:16.341236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:19:16.341236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:19:16.341236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[105b7575-594d-4442-8561-f8a7b21abdc1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:16.343086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:16.344869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:16.348878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[833bfb7f-7e98-4208-b8d7-7c3a710fe4f0]: (4, ['qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.348878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:16.348878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:16.348878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:19:16.348878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:16.348878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7efaa7c-bc34-4f47-abc1-a9e7c372a1ed]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.349980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a01e2ed-ad00-4818-98a6-8d1c4edad749]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.351163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6310ca9b-2cc0-4331-9d58-ce31d3a8012e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.369060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64fb7854-6c95-4334-884f-22167676c505]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 205230, 'reachable_time': 20436, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 111795, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.374690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:19:16.374955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc409438-8f8f-47e6-9cac-8808ef744775]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:16.376019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "router1", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43" Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:34Z", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:52Z", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:50:2e", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 34, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:52Z", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:55Z", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fa0ff57-49b7-4c01-8fdc-8855d809fdc0", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b71076a-940c-4868-a739-6596faed3a10", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0f:4c:08", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 27, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 26 Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:44Z", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:19:13Z", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/26", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": "c8ea00fe-9c19-4351-88b9-0cff9ceac3e0" Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:19:16.378674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id cbe84ffb-c3d7-467b-807d-482c5cb0c868. Wait time elapsed: 22.806 Jul 01 09:19:16.393260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3, priority 2, update_id cbe84ffb-c3d7-467b-807d-482c5cb0c868. Wait time elapsed: 22.806 Jul 01 09:19:16.393260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:19:18.030843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.652s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:19:18.032421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:19:18.033535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a5215a0-df88-4c5c-a926-a3c7b3bf5f79), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:19:18.034005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:18.034429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:18.037147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36adea01-8daa-4042-825c-863be635b69a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.070358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[629d79bd-4142-4787-8703-04be5bcedd22]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.104117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16f3a220-d8d5-4226-b411-06294ef2f931]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.162126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93c83568-82ed-4cc0-aa0d-204fda96fd2b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 111811, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.193984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a479ead7-2996-49ab-ab2c-80e15aa2bbd2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.230515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83ef8565-6f1f-44cf-a8a6-f8480103f1ed]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.265133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd77b234-95a0-4eac-ab08-ec3fcfe67cb2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.300363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0a5cb17-b420-44f8-a6ca-72b6427bcc7b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.301159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:19:18.301626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:19:18.302052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:19:18.302474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fa0ff57-49b7-4c01-8fdc-8855d809fdc0) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:19:18.329082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:19:18.329602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c88a9e80-8bfa-4252-85cf-092dd28c0f83]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.332584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58c4e73d-d6ba-4ac6-af17-b5e9a4e8a1c5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.334390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fa0ff57-49, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:18.375465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fa0ff57-49, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:18.376398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:18.376935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:18.377490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fa0ff57-49, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:18.378031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fa0ff57-49, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fa0ff57-49b7-4c01-8fdc-8855d809fdc0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0f:4c:08'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:19:18.440847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0349294-e80a-4380-bd75-a8cadd5b3319]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.508346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f198d13c-efb0-48e7-9a90-69bc76c5d431]: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:19:18.508346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:19:18.508346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:19:18.508346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:19:18.508346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:19:18.508346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:19:18.508346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:19:18.508346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:19:18.508346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:19:18.508346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:19:18.508346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:19:18.508346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:19:18.508346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:18.509309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f198d13c-efb0-48e7-9a90-69bc76c5d431]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:18.510447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:18.512218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:18.514500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f24cad72-599d-43cd-ad23-4b7148fb5e60]: (4, ['qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.514500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:18.514500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:18.514500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:19:18.514500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a5215a0-df88-4c5c-a926-a3c7b3bf5f79.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:19:18.517492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b96c3db-fb2f-45b1-aae7-5dcf86ef9ba8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.521733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcd92560-d106-4f3a-a298-f01db210d47e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.522588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8098455-9d3d-40df-af58-534ecb48544f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.567832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ddc6349-c043-4c73-ad17-68c9acc3f4c6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 205442, 'reachable_time': 28280, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 111830, 'error': None, 'target': 'qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.574218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:19:18.575260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abb46d3d-030e-46cc-9534-9eaa8371829e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fa0ff57-49 not found in namespace qrouter-0a5215a0-df88-4c5c-a926-a3c7b3bf5f79. Jul 01 09:19:18.576562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "router1", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43" Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:34Z", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:52Z", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3dfbf632-6168-4363-ac2e-5d4a6453ac60", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:50:2e", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 34, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.43", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:52Z", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:18:55Z", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fa0ff57-49b7-4c01-8fdc-8855d809fdc0", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b71076a-940c-4868-a739-6596faed3a10", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0f:4c:08", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a5215a0-df88-4c5c-a926-a3c7b3bf5f79", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 27, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 26 Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:18:44Z", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:19:16Z", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c8f2172a0a944560bcee31be13e1a045", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "86aaeabd-5b88-409a-9455-26ae1b48f9c9", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/26", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": "c8ea00fe-9c19-4351-88b9-0cff9ceac3e0" Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:19:18.579689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 0a5215a0-df88-4c5c-a926-a3c7b3bf5f79, action 3 Jul 01 09:19:34.891271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:20:14.224497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:20:14.225162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:20:14.496775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:20:14.497074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:20:14.893889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:20:54.895785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:21:14.226733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:21:14.226733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:21:14.497277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:21:14.498136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:21:34.897919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:22:14.227480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:22:14.228129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:22:14.497779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:22:14.498317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:22:14.899305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:22:54.900631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:23:14.227309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:23:14.228017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:23:14.498603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:23:14.499117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:23:34.902152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:24:14.227925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:24:14.229358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:24:14.499337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:24:14.499629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:24:14.903526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:24:54.905054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:25:14.228426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:25:14.229016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:25:14.500604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:25:14.500977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:25:34.906116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:25:57.616564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-93a9ecbb-084a-4c23-8bf8-795e28081441 admin admin] Got routers updated notification :['4da8db80-0643-4392-ad3e-03f5c3888347'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:25:57.617241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 1, update_id fa6a959f-0fb2-42e3-96a6-c8d1e1bfdf36. Wait time elapsed: 0.000 Jul 01 09:25:57.617241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 1, update_id fa6a959f-0fb2-42e3-96a6-c8d1e1bfdf36. Wait time elapsed: 0.001 Jul 01 09:25:57.617703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:25:58.041509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-e4f3e0a2-2631-4e02-a65c-8546260701d5 admin admin] Got routers updated notification :['9974ba83-7306-4b67-8057-d6ab904a1c00'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:25:58.042698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 1, update_id 8acfcf1d-3af9-4cc1-85e9-2d96ae9cf6de. Wait time elapsed: 0.001 Jul 01 09:25:58.043501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 1, update_id 8acfcf1d-3af9-4cc1-85e9-2d96ae9cf6de. Wait time elapsed: 0.001 Jul 01 09:25:58.044015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:25:59.484602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.867s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:25:59.485281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:25:59.487146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:25:59.487146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:25:59.487146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:25:59.488255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57aa2a8d-6785-4c91-8b17-8b7c3de98f58]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:25:59.513956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[395e3ee6-7ab3-44b5-adc3-996d50d83dd3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:25:59.557820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7c41b55-7812-49a6-b805-2df72927ce4c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:25:59.609947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9afc0b4e-9545-43c4-b025-176ba0adb6d6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 125449, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:25:59.644926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e01f3122-549f-4304-b727-a424a984e060]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:25:59.680357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac0b992a-831c-42e0-b7ba-0832e66ec45b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:25:59.715076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56cca4a1-7cf1-4f89-8141-bdb89d067df8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:25:59.750700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea6efe21-a9ee-4dd0-a35a-ba82eb5164e5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:25:59.751516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:25:59.751891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:25:59.752288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:25:59.780564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcd435c2-5568-4f36-b5b6-1bea1bba4aa8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 245587, 'reachable_time': 29904, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125455, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:25:59.781861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:25:59.781861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:25:59.782485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:25:59.782857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:25:59.783296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:25:59.784257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:25:59.784257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '37d4d8f8-7374-4b3b-b030-66d7cccb36f0', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:c8:0e:04', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '4da8db80-0643-4392-ad3e-03f5c3888347', 'device_owner': 'network:router_gateway', 'standard_attr_id': 407, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.113', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:25:54Z', 'updated_at': '2022-07-01T09:25:58Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-37d4d8f8-73), ns(qrouter-4da8db80-0643-4392-ad3e-03f5c3888347) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:25:59.810537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:25:59.811137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf1755c5-ea3a-4129-8f54-48c8c314d167]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:25:59.814511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd639d43-8b83-46f3-892e-be8ad6e1c1b3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:25:59.816410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-37d4d8f8-73, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:25:59.816944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:25:59.818324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-37d4d8f8-73, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:25:59.819880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-37d4d8f8-73, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:25:59.820659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-37d4d8f8-73, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:25:59.821255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-37d4d8f8-73, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:25:59.821840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-37d4d8f8-73, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37d4d8f8-7374-4b3b-b030-66d7cccb36f0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:0e:04'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:25:59.856673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b34f19c1-8fb6-4718-8290-aa1a5d303606]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:25:59.893421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c03350db-dcb8-4bc1-b8df-9d7b1735bfde]: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:25:59.893421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:25:59.893421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:25:59.893421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:25:59.893421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:25:59.893421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:25:59.893421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:25:59.893421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:25:59.893421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:25:59.893421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:25:59.893421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:25:59.893421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:25:59.893421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:25:59.894852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c03350db-dcb8-4bc1-b8df-9d7b1735bfde]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:25:59.895664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:25:59.897462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:25:59.950244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05000760-e5f9-4359-b62c-2250af1ad9de]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:00.342613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a44bdbff-5624-48e1-a96d-170b7f1b07bc]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:00.370044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04b46f3d-dad7-41de-9548-35c824ad5fc4]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:00.766683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a86c8e0-f395-4b4b-9707-80328e211a8a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:00.767280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:26:00.767646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:00.767935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:00.768389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" released by "process_external" :: held 0.986s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:00.769718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:00.772968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:00.775934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a76456f6-11f8-44cd-8508-49fbbacbfacc]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:00.775934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:00.775934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:00.775934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:00.776264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:00.778397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0505a235-b85a-4c40-b0a7-d0aff2600e20]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:00.779260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1be87cff-8550-48ce-bf8d-3dae299d569f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:00.780407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b697772e-2b02-48bd-8cac-b20eb394087d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:00.816933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52fc4a8c-1591-4e60-909b-b70718ae0f95]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 245587, 'reachable_time': 29904, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125516, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:00.827574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:00.828270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[487ca132-f2c5-4696-b8ad-3cb8a5ede378]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:00.829831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:25:55Z", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:25:58Z", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:26:00.835426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:00.846988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id fa6a959f-0fb2-42e3-96a6-c8d1e1bfdf36. Wait time elapsed: 3.225 Jul 01 09:26:00.846988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id fa6a959f-0fb2-42e3-96a6-c8d1e1bfdf36. Wait time elapsed: 3.226 Jul 01 09:26:00.846988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:01.893648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8f8f3e53-2d96-4f11-8448-3ecf75ec2162 tempest-RoutersAdminNegativeTest-961178275 tempest-RoutersAdminNegativeTest-961178275-project-admin] Got router deleted notification for 6d23332c-a617-4389-8bad-408181c8cfe9 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:26:01.894774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 6d23332c-a617-4389-8bad-408181c8cfe9, action 1, priority 1, update_id cf6556da-7e8e-458a-ba30-73f95163e1a4. Wait time elapsed: 0.001 Jul 01 09:26:01.895266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 6d23332c-a617-4389-8bad-408181c8cfe9, action 1, priority 1, update_id cf6556da-7e8e-458a-ba30-73f95163e1a4. Wait time elapsed: 0.001 Jul 01 09:26:01.895774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 6d23332c-a617-4389-8bad-408181c8cfe9 was not found. Performing router cleanup Jul 01 09:26:01.897161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05dc9d14-2385-4918-a8fb-c0a85b6b09db]: (4, ['qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:01.898635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:01.899233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:01.899824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 6d23332c-a617-4389-8bad-408181c8cfe9, update_id cf6556da-7e8e-458a-ba30-73f95163e1a4. Time elapsed: 0.005 Jul 01 09:26:02.011765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.169s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:02.012314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:02.013716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:02.014247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:02.014758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:02.016449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9ec367b-c49e-411d-a158-069f415c002b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:02.042867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[816c7312-9044-4a20-8fb7-b264cea7befd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:02.073508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a608f1d4-bf09-4b7f-8ccc-f41bbe518a8a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:02.119091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b66516d9-a4c5-4b40-a382-8e9c84ded115]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 125523, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:02.930567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10943d14-9257-4eb5-b7f3-27b95d254ed4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:02.957129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6a8afff-900b-489d-b267-6c3a85dd2bde]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:02.985577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b814d0a4-1bf7-4717-b0d1-0b61ef0d854e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.021955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[977bb1e9-cf4b-47ed-88e4-758fe5b23faf]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.022977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:03.023298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:03.023670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:03.049659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7cddf5bf-aa19-4be5-9d1a-bcda5ed16c4d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 245838, 'reachable_time': 32952, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125532, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.050539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:03.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:03.052021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:03.052021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:26:03.052021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:03.052406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:03.052767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '37d4d8f8-7374-4b3b-b030-66d7cccb36f0', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:c8:0e:04', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '4da8db80-0643-4392-ad3e-03f5c3888347', 'device_owner': 'network:router_gateway', 'standard_attr_id': 407, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.113', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:25:54Z', 'updated_at': '2022-07-01T09:25:58Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-37d4d8f8-73), ns(qrouter-4da8db80-0643-4392-ad3e-03f5c3888347) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:26:03.078333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:03.078805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e6c208e-6b64-4670-b210-3197a6e98a7a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.082926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c491615a-72a6-47c3-8fba-9078aa9a665a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.087021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-37d4d8f8-73, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:03.122819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-37d4d8f8-73, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:03.124485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-37d4d8f8-73, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:03.125853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-37d4d8f8-73, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:03.126455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-37d4d8f8-73, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:03.127543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-37d4d8f8-73, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37d4d8f8-7374-4b3b-b030-66d7cccb36f0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:0e:04'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:03.180046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7a1b381-9a7a-4ea8-a359-b69f8b31b95f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.221537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b0fa6c87-6a70-4faf-8b71-209740befaa6]: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:03.221537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:03.221537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:03.221537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:03.221537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:03.221537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:03.221537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:03.221537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:03.221537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:03.221537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:03.221537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:03.221537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:03.221537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:03.222692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0fa6c87-6a70-4faf-8b71-209740befaa6]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.223496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:03.224170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:26:03.250844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1be6a99d-338a-4ac6-a494-e625c307d0ef]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.467707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88b22d71-ee12-479d-aad1-729de1835b5d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.503455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb2a63ce-9912-452e-9649-ecd87fe77059]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.728621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e67b105-6d42-4bf3-9478-88199dedf50c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.732205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:26:03.733499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:03.734076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:03.734607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" released by "process_external" :: held 0.683s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:03.735685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:03.742576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:03.747472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26a7768d-1b7c-4e9a-b5fc-2207e7a37fd1]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.748386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:03.748922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:03.749340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:03.749926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:03.751866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ab4c982-248f-48d6-a2f5-265aebd93b5b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.753095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd89b297-144f-4499-8363-df35d3d5f9f9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.754222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5b9a5d8-36d9-4482-8e61-e33101585eb2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.777403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.733s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:03.778710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:03.785413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:03.787742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:03.788807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:03.790391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68b4ffe0-f28c-4479-b12a-a5edf7022c13]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 245838, 'reachable_time': 32952, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125549, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.793665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a3caf09-8e10-494c-b538-524f586b0803]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.814340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:03.814885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c3fa178-87df-423e-a351-fccf89290b77]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:03.818404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:25:55Z", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:25:58Z", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:26:03.828238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:03.835863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a7ff7c3-6a20-41a0-bf37-b1219d92d936]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.837685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id fa6a959f-0fb2-42e3-96a6-c8d1e1bfdf36. Wait time elapsed: 6.222 Jul 01 09:26:03.837942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id fa6a959f-0fb2-42e3-96a6-c8d1e1bfdf36. Wait time elapsed: 6.222 Jul 01 09:26:03.838513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:03.865101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec2bfe06-2b71-4dbc-bde3-b41bbadbb84d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.914376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9879c23-a187-481d-8a74-14f8b31f11b2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 125553, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.937042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0aeb4c3c-7bec-4612-a7f6-f8098f4aa0b2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.969932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b5cdba3-6fcd-4b80-bf33-9bb0f2457682]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:03.997231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe180c76-e0e6-4617-a322-05b4499bf552]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.046443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3485b75-5546-42e7-979e-6686bcd080dd]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.047471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:04.049569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:04.050182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:04.080197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[067687cf-a04f-434b-9c6d-61fce2d31f61]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 246017, 'reachable_time': 25840, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125562, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.085731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:04.086696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:04.087237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:04.087694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:26:04.088125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:04.088484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:04.089394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'fdf456c3-4b9b-4ddb-95a4-0e9aa788e001', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:42:56:61', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '9974ba83-7306-4b67-8057-d6ab904a1c00', 'device_owner': 'network:router_gateway', 'standard_attr_id': 412, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.132', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:25:54Z', 'updated_at': '2022-07-01T09:26:01Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-fdf456c3-4b), ns(qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:26:04.111475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-fdf456c3-4b not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:04.111902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37540d8e-db22-4fd7-9602-aadffa9e1701]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.117485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3f565e0-da78-4f73-8d0e-ff2c89fc5138]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.118401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-fdf456c3-4b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:04.118723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:26:04.120278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-fdf456c3-4b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:04.120953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-fdf456c3-4b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:04.121346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-fdf456c3-4b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:04.121979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-fdf456c3-4b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:04.122368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-fdf456c3-4b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'fdf456c3-4b9b-4ddb-95a4-0e9aa788e001', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:56:61'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:04.200299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79c5daf3-232c-43f3-809f-1c1619aa0379]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.244564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9c4e5264-3e9a-4f3a-a44d-6108824af59e]: Network interface qg-fdf456c3-4b not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:04.244564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:04.244564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:04.244564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:04.244564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:04.244564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:04.244564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:04.244564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:04.244564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:04.244564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:04.244564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:04.244564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:04.244564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-fdf456c3-4b not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:04.245435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c4e5264-3e9a-4f3a-a44d-6108824af59e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-fdf456c3-4b not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.245435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:04.246235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:26:04.294353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8bf8445-492d-4918-9773-4c6760bfcaa2]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.502898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[504537dc-6d31-4268-b5e5-f58da017d8bd]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.551704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[662c1a59-082d-41ea-afae-635d4ba128a0]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.775534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c81409b7-9a45-4a5e-bbc7-60023c360692]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.776517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:26:04.777053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:04.777907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:04.778457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" released by "process_external" :: held 0.691s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-fdf456c3-4b not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-fdf456c3-4b not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-fdf456c3-4b not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:04.779613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-fdf456c3-4b not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-fdf456c3-4b not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:04.783662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:04.788422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9289ce9d-0f45-48ca-8e15-00f31c30cf9b]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.789545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:04.790167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:04.790601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:04.791272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:04.793526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75233845-abc4-4099-831c-2f569ba7e0bb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.795545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6822bb89-8e62-45c5-9cd1-0cc96859572b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.796817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45be2791-b53b-4875-94c4-56d52c079270]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.828115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[164d9bbf-1995-47ac-8e4d-5954f5447615]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 246017, 'reachable_time': 25840, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125582, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.837263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:04.837801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88979492-052e-4b72-94c1-937592f4c9c8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-fdf456c3-4b not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-fdf456c3-4b not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:04.839174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:25:55Z", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:01Z", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:26:04.843544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:04.854991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 8acfcf1d-3af9-4cc1-85e9-2d96ae9cf6de. Wait time elapsed: 6.813 Jul 01 09:26:04.855601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 8acfcf1d-3af9-4cc1-85e9-2d96ae9cf6de. Wait time elapsed: 6.813 Jul 01 09:26:04.856271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:05.926042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-00003a92-a61d-41db-bdbb-efa6d20848c2 admin admin] Got routers updated notification :['4da8db80-0643-4392-ad3e-03f5c3888347'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:26:07.373210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-814ab86b-949b-45fc-9273-cc3f16966963 admin admin] Got routers updated notification :['9974ba83-7306-4b67-8057-d6ab904a1c00'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:26:07.766600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.910s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:07.766787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:07.768164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:07.768701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:07.769403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:07.772081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8c69f3c-e37e-43b5-8936-56820ec28b3e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:07.802228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[316e5940-d093-40fc-97ba-72c6a42bb9d0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:07.846449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c864045-8a40-4b8c-b328-42fab008373f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:07.907973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de23896d-aad8-422a-bd12-8f06ad08034e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 125640, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:07.949374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d819835-0c63-452a-944f-94ed3e397037]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:07.991826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14745cfa-c23c-4736-8eb1-501525d08481]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:08.029765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4a8d500-ca14-4868-843f-ae1a1d96fd6b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:08.071170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[353a3aee-9816-47dc-97bb-859023edca85]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:08.071962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:08.072264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:08.072626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:08.072968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:08.115675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:08.116731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66c43bc3-d9cb-416c-a753-c1c7e6f7160e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:08.120360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f148b53-e21d-48af-82b5-c47a935e37f9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:08.121377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:08.121858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:26:08.122655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:08.123418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:08.123894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:08.124331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:08.124763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:08.176056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[533ed648-bb7e-46df-bd11-4cee10989c6a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:08.243372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bd1940a9-f96d-482c-8899-fe3ebe00930c]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:08.243372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:08.243372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:08.243372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:08.243372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:08.243372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:08.243372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:08.243372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:08.243372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:08.243372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:08.243372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:08.243372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:08.243372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:08.244781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd1940a9-f96d-482c-8899-fe3ebe00930c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:08.246179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:08.248978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:08.252669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b33dd97-95a5-4e03-ae3d-4fc090afca15]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:08.253602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:08.254146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:08.254566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:08.255103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:08.256303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[884e4a07-654d-413c-b260-fb1d704d65f5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:08.260092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39cb4f52-a940-489b-b5bf-043416012617]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:08.261414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[062839ce-aa2a-4847-b62a-05c5f8edf0d8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:08.301861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9efd56ae-1c43-4775-82b9-05dfdd0f120e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 246417, 'reachable_time': 16868, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125658, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:08.311737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:08.312282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b06e61b-9a7b-4e36-8904-a90f702f2479]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:08.313552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:01Z", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:06Z", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:08.317285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:08.329642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 1, update_id 45974039-8826-4afd-ad22-e2d1e679b032. Wait time elapsed: 0.949 Jul 01 09:26:08.329642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 1, update_id 45974039-8826-4afd-ad22-e2d1e679b032. Wait time elapsed: 0.949 Jul 01 09:26:08.332605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:11.439985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-31fe24dd-98e7-47e8-aead-a16c00cb4277 tempest-RoutersAdminTest-1421187323 tempest-RoutersAdminTest-1421187323-project-admin] Got routers updated notification :['7d80a9c7-6aa4-4b4e-859d-03c7d434020b'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:26:11.440469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7d80a9c7-6aa4-4b4e-859d-03c7d434020b, action 3, priority 1, update_id f89d29de-df40-47c7-b994-37fa57e1aa0e. Wait time elapsed: 0.000 Jul 01 09:26:11.440469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7d80a9c7-6aa4-4b4e-859d-03c7d434020b, action 3, priority 1, update_id f89d29de-df40-47c7-b994-37fa57e1aa0e. Wait time elapsed: 0.000 Jul 01 09:26:11.440704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:11.909462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 8.071s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:11.909730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:11.910888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:11.911023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:11.911228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:11.912990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba95feb4-b60b-47cc-8ea7-7bd7bc4c3db2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:11.932790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df9ae6fc-35b5-49a1-807f-c07320a768b0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:11.980063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb93d7aa-f50e-4d44-a318-db9f5c96ce81]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.036089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dcf52687-524a-4e3e-8de4-ec841f49704c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 125684, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.077671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bdd03fcc-fa9d-4272-a901-fa093c30334f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.110420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3089937-fa36-4835-8544-f2e4afd618e3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.145776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7aa3386d-c2b3-4c97-a036-2448cb3069c3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.187213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e3a6b2f-5988-4bb4-8d17-b00f56d8eb56]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.187809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:12.188083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:12.188347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:12.188614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:12.221923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:12.222529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07ec40e8-2abc-40b9-82cd-03f6cbab515f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.226529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9847d5f-0fb3-4b2f-83f8-8073133dbe6b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.227722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:12.228161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:26:12.229073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:12.229872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:12.230403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:12.230947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:12.231493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:12.282687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aad25421-63f2-41f8-a561-998c4312710a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.332819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5209dd7f-d6a3-4bd5-81d3-595000a9a97a]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:12.332819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:12.332819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:12.332819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:12.332819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:12.332819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:12.332819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:12.332819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:12.332819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:12.332819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:12.332819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:12.332819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:12.332819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:12.334249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5209dd7f-d6a3-4bd5-81d3-595000a9a97a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:12.336192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:12.338835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:12.342724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82fe6ea7-bcc8-4959-a90c-e2596f7d94ca]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.343709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:12.344256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:12.344677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:12.345240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:12.346499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3a2628f-f670-48e8-a6f1-7babde4cd8fd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.347664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bebc806-7e5a-4407-8379-14956109469a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.348811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b54fac88-51f8-4b6e-919b-247eddcb0931]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.384354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[daae4ce7-f65c-4c74-a4fd-e2b7cc62dfa7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 246830, 'reachable_time': 31144, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125703, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.386949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:12.387450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[081cb4be-c485-4157-ad1f-577ec430c7f5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:12.389587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:08Z", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:12.393414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:12.406240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 1, update_id c2f47779-0b08-4fcc-81c9-1ac989d09e49. Wait time elapsed: 6.478 Jul 01 09:26:12.406240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 1, update_id c2f47779-0b08-4fcc-81c9-1ac989d09e49. Wait time elapsed: 6.478 Jul 01 09:26:12.406240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:12.745945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.414s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:12.747160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:12.751872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:12.752394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:12.752886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:12.760375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a99983c-969e-42f8-88d0-746df8fccc82]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.791723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1459666e-8d98-4860-83eb-6d9ad3320aaf]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.824208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7d9f6f5-af5d-401c-9674-5673ac37e246]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.867893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5254fbc-e0c2-4487-832e-85781119d598]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 125707, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.901505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b5a0f9a-e264-464d-b9ce-0a0ca08dc5c6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.938164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f7f36ab-8aa6-48b4-b6f1-a116cd8b67b9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:12.972648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06b4fcdd-e446-4094-8354-377cd0558ad8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.009825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c506fa4a-29f5-491c-9ff2-1841e10e19e3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.010527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:13.010901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:13.011265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:13.011663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:13.044756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:13.045118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88835534-ea86-42b8-9756-802cb84ac8d2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.048506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f5e54e3-f183-4284-810c-cc0ad95db165]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.049732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:13.087446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:13.088333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:13.089154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:13.089751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:13.090346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:13.144076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71f579c8-02b1-4aed-9fb7-cde24e6a2104]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.171517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[35967d65-a0d1-4a4e-9e63-1594e1f6ef5b]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:13.171517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:13.171517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:13.171517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:13.171517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:13.171517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:13.171517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:13.171517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:13.171517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:13.171517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:13.171517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:13.171517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:13.171517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:13.172573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35967d65-a0d1-4a4e-9e63-1594e1f6ef5b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:13.172699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:13.174535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:13.176905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3f86439-db46-454a-a997-aabfc3664dbf]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.176905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:13.176905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:13.176905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:13.176905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:13.178974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[debc8a97-6bc4-4ad9-9671-b0ddec881802]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.180333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9319f38c-cf9d-4e03-8b5b-69ba31a2da57]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.188015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44b5d3a9-fb39-45b7-ab48-e6397515dba1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.227778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[524efd8b-d2a0-44d9-bd53-c59a97f99524]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 246912, 'reachable_time': 15320, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125725, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.236006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:13.236420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c80b55b-69dd-40d0-8493-6fd8bb9a1060]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:13.237563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:01Z", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:06Z", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:13.242375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 8acfcf1d-3af9-4cc1-85e9-2d96ae9cf6de. Wait time elapsed: 15.199 Jul 01 09:26:13.257831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 8acfcf1d-3af9-4cc1-85e9-2d96ae9cf6de. Wait time elapsed: 15.200 Jul 01 09:26:13.257831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:13.690810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.250s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:13.691340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7d80a9c7-6aa4-4b4e-859d-03c7d434020b info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:13.692556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7d80a9c7-6aa4-4b4e-859d-03c7d434020b), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:13.692984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:13.693626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:13.695154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[297efd2f-06fe-4f82-b6aa-167fa94f379c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.713392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8afd93f6-5189-465b-b222-9625195b9d50]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.740317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc1b72b5-af0e-4b93-a112-ebdfc11d2f87]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.809089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03e04c94-89cb-4f6b-98b5-7c5038a34fd5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 125728, 'error': None, 'target': 'qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.843743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f913e0c8-a2bf-4cf0-b356-c8e3a2787b68]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.876828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c1152fc-9968-4066-a46b-7881c0133b5a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.925067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64d7342b-81fe-49c4-9e52-f96961894caf]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.961582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6059ba14-212b-4398-a83a-2c0fd8ce7af5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:13.962977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:13.963472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:13.963898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7d80a9c7-6aa4-4b4e-859d-03c7d434020b {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:14.001065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8eed4e8-7136-4fab-9a15-df38c32b7b39]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 247007, 'reachable_time': 16872, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125738, 'error': None, 'target': 'qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:14.002491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:14.003084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:14.003807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:14.004315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:26:14.050821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3809558-b58b-4377-9c15-f36fc29d703a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 247007, 'reachable_time': 16872, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125745, 'error': None, 'target': 'qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:14.052184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:14.052801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:26:14.083263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d0279b5-a0ee-4868-a0f1-56fabb75c737]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.057420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad80f562-6dc3-489d-8268-33b72958210a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.063443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:26:15.064154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:15.064697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:15.065171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:15.065571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:15.110725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fe5f6d3-8b2e-4ac9-a7c5-ae6b6407592f]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.199288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.955s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:15.199751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:15.200727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:15.201141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:15.201834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:15.203853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4951980f-bc93-4d51-bd2e-cacf3006e9ea]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.233255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44929e84-59d2-4370-b15a-09e3f8f0d1ba]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.275166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[146e9705-e6fa-445f-8bf7-3dae8fcd6cf2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.320234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[002289ea-8d07-4349-8fdb-94260492de29]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.320887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:26:15.321312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:15.321704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:15.322412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" released by "process_external" :: held 1.318s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:26:15.323028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:15.323319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:26:15.323831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:15.324389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:26:15.340435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[866db894-9024-40e8-96c5-efd56e6be498]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 125759, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.370204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8ba1fb3-7b64-4467-83a6-56cc44df34e0]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.380070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40c73c68-46cf-4a48-9f85-89af280d41d2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.436196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ea83fb3-4de7-437a-a4b3-885982962503]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.450438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6218f325-dc8c-4d47-be83-50fd09f78c58]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.481061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c3813f1-a652-4c6b-a4ff-f4372efe3eba]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.495385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d98414f3-348b-4c92-b724-6c2db434ea19]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.535733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c6c7faf-3c90-45a0-a154-df77e48ebd84]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.536749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:15.537121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:15.537998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:15.538610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:15.580812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:15.582540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f98d3e78-b490-4646-86e5-9d5c80276c17]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.582955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c620327-786d-4b30-b570-c64adf995b06]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.584010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:26:15.586777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:15.587137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:15.587565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" released by "process_address_scope" :: held 0.264s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:26:15.587998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (7d80a9c7-6aa4-4b4e-859d-03c7d434020b), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:15.589002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:15.589338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" acquired by "apply_metadata_nat_rules" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:26:15.590354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:15.590951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:26:15.591924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7dd273b9-576f-4a22-9471-b9885afdeb0f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.593887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:15.635797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:15.636713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:15.637305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:15.638441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:15.639022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:15.652932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21c7210f-310c-4ac3-b7a0-07fa868e8b60]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.697718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d86d23e-6cd8-43a5-a33b-1d821ebaaa90]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.749064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b35631ec-2a74-4467-b885-bb70cc2c4a06]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.758422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5205d77f-25ce-4dc8-a4ad-ccf0683b8faa]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:15.758422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:15.758422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:15.758422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:15.758422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:15.758422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:15.758422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:15.758422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:15.758422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:15.758422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:15.758422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:15.758422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:15.758422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:15.760010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5205d77f-25ce-4dc8-a4ad-ccf0683b8faa]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:15.761301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:15.763863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:15.765863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52a07c09-914d-4892-9a20-5f7e48e5de5d]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.765863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:15.766393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:15.766492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:15.766797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:15.769756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82b5e56c-e3ad-490e-a15d-fe22ad3e354f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.770839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64d623d6-a4a4-4c9b-bbdd-a164561d5626]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.772044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1417bba-c108-4d01-978d-66908ff07a51]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.789724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[611c9e4b-e778-4c99-8de8-39cb3e2539b6]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:15 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:26:15 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.810591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.405s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:15.811304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:15.812571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:15.813553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:15.814330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:15.817072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[294d571c-6eec-48b0-9ef2-11accd7289fa]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 247160, 'reachable_time': 16348, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125786, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5f7ffd8-f1ab-451d-a54e-44b74435a5af]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:15.826196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:13Z", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 45974039-8826-4afd-ad22-e2d1e679b032. Wait time elapsed: 8.452 Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 45974039-8826-4afd-ad22-e2d1e679b032. Wait time elapsed: 8.452 Jul 01 09:26:15.832773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:15.853309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfffd5cf-e4f5-4f28-8ea9-41cab1e2c14f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.864806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[937d9553-d973-4790-80db-009d34a6a2af]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.906984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcef61ff-21d0-406c-8041-0c893753898d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.913548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49cae762-5814-4bcb-b213-dd90fcafb211]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:15.914167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:26:15.914540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:15.914857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:15.915204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b" released by "apply_metadata_nat_rules" :: held 0.326s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:26:15.915829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7d80a9c7-6aa4-4b4e-859d-03c7d434020b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7d80a9c7-6aa4-4b4e-859d-03c7d434020b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-7d80a9c7-6aa4-4b4e-859d-03c7d434020b Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/7d80a9c7-6aa4-4b4e-859d-03c7d434020b.pid.haproxy Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 7d80a9c7-6aa4-4b4e-859d-03c7d434020b Jul 01 09:26:15.916865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:26:15.920101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/7d80a9c7-6aa4-4b4e-859d-03c7d434020b.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:26:15.945700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_rootwrap.client [-] Popen for ['sudo', '/usr/local/bin/neutron-rootwrap-daemon', '/etc/neutron/rootwrap.conf'] command has been instantiated {{(pid=106231) _initialize /usr/local/lib/python3.8/dist-packages/oslo_rootwrap/client.py:74}} Jul 01 09:26:15.985811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-a8f11f2e-a7b7-420c-9f57-d4edbb897eb5 tempest-RoutersAdminTest-1421187323 tempest-RoutersAdminTest-1421187323-project-admin] Got router deleted notification for 7d80a9c7-6aa4-4b4e-859d-03c7d434020b {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:26:16.002315 dv-neutron-848393-1-hyperv sudo[125793]: ubuntu : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/usr/local/bin/neutron-rootwrap-daemon /etc/neutron/rootwrap.conf Jul 01 09:26:16.004960 dv-neutron-848393-1-hyperv sudo[125793]: pam_unix(sudo:session): session opened for user root by (uid=0) Jul 01 09:26:16.052644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c56e363-09a6-41aa-b1e9-4f940191d38d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 125792, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.113465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7808371f-4ecf-4a1e-8020-9dd3f84e827f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.148807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b5fe72b-d00f-464a-8109-a8bc677d6d54]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.180798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b216609-9a7b-4ef8-9df5-e73aa753ebe4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.218160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e714e1b3-dc4b-4a91-b994-bec0f4f393e9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.218851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:16.219196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:16.219540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:16.219873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:16.265614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:16.265902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f97658bd-5c9b-4b60-bb33-7496d186f065]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.268767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b2cfada-fefe-4d27-9958-66b7ae187e15]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.272215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:16.303916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:16.305520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:16.306124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:16.306730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:16.307345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:16.332371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e221c21-8485-48a1-a901-d5e619dc958e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.390622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[30d58f02-695e-4238-bad9-ea2a827acfcf]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:16.390622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:16.390622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:16.390622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:16.390622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:16.390622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:16.390622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:16.390622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:16.390622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:16.390622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:16.390622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:16.390622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:16.390622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:16.391629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30d58f02-695e-4238-bad9-ea2a827acfcf]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:16.391972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:16.393522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:16.395594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4e7608b-16cd-4a36-83a6-8fe529898c97]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.395594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:16.395594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:16.395594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:16.395878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:16.398315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e95c34b5-e6a4-450a-a98b-cd6eb74b1bf0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.399413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee6bbf34-fa38-40be-86e5-f76fc2a222b2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.400394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59b3fc1f-3819-4f97-8c57-2ccdce27a9d1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.445465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c980ef79-e0c0-4276-8907-311b9e55524d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 247231, 'reachable_time': 37480, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125814, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.451974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:16.452232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[513d4cfb-6fba-4933-82f6-ffcffb82f4ee]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:16.453321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:08Z", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:16.456088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id fa6a959f-0fb2-42e3-96a6-c8d1e1bfdf36. Wait time elapsed: 18.839 Jul 01 09:26:16.472946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id fa6a959f-0fb2-42e3-96a6-c8d1e1bfdf36. Wait time elapsed: 18.839 Jul 01 09:26:16.472946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:16.921244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO oslo_rootwrap.client [-] Spawned new rootwrap daemon process with pid=125793 Jul 01 09:26:17.036439 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-7d80a9c7-6aa4-4b4e-859d-03c7d434020b[125822]: Proxy listener started. Jul 01 09:26:17.051446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 7d80a9c7-6aa4-4b4e-859d-03c7d434020b add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:26:17.052054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 7d80a9c7-6aa4-4b4e-859d-03c7d434020b, update_id f89d29de-df40-47c7-b994-37fa57e1aa0e. Time elapsed: 5.612 Jul 01 09:26:17.052469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7d80a9c7-6aa4-4b4e-859d-03c7d434020b, action 1, priority 1, update_id 28ffcc1c-8e7e-437c-8370-091b081e318f. Wait time elapsed: 1.066 Jul 01 09:26:17.052870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7d80a9c7-6aa4-4b4e-859d-03c7d434020b, action 1, priority 1, update_id 28ffcc1c-8e7e-437c-8370-091b081e318f. Wait time elapsed: 1.066 Jul 01 09:26:17.055338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 7d80a9c7-6aa4-4b4e-859d-03c7d434020b delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:26:17.055916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (7d80a9c7-6aa4-4b4e-859d-03c7d434020b), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:17.091103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55274c66-93da-45ff-93cb-ac90a07ada97]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:17.093049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad60f0b4-ab31-456f-a3c9-86d33d52c6d6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:17.094178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73ed7646-ec7f-4c46-810b-7453c31e5432]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:17.094774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 7d80a9c7-6aa4-4b4e-859d-03c7d434020b {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:26:17.095805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f97cfda2-e84a-4d95-9a16-65460f865785]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:17.122181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b834574a-428f-4285-ba14-dc1559f75c9e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 247007, 'reachable_time': 16872, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125828, 'error': None, 'target': 'qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:17.122947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:17.123248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:17.153485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4f066c1-7fa1-4bed-b4b8-8a52018d683a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 247007, 'reachable_time': 16872, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125829, 'error': None, 'target': 'qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:17.154557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 7d80a9c7-6aa4-4b4e-859d-03c7d434020b {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:26:17.155057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7d80a9c7-6aa4-4b4e-859d-03c7d434020b.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7d80a9c7-6aa4-4b4e-859d-03c7d434020b.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:17.155402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7d80a9c7-6aa4-4b4e-859d-03c7d434020b.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7d80a9c7-6aa4-4b4e-859d-03c7d434020b.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:17.155654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 7d80a9c7-6aa4-4b4e-859d-03c7d434020b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:17.156860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 7d80a9c7-6aa4-4b4e-859d-03c7d434020b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:26:17.157909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fc4bb29-712e-41f3-9dd3-721b53248e2e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:17.184333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b13db46f-139d-4676-b7e6-121290da5d68]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 247007, 'reachable_time': 16872, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125830, 'error': None, 'target': 'qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:17.190204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7d80a9c7-6aa4-4b4e-859d-03c7d434020b deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:17.190204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a04223e-ea53-46c7-a34a-a9e06590644d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:17.190204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 7d80a9c7-6aa4-4b4e-859d-03c7d434020b delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:26:17.190204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (7d80a9c7-6aa4-4b4e-859d-03c7d434020b), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:17.190204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:17.190204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:17.190204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:17.190204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:17.190204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 7d80a9c7-6aa4-4b4e-859d-03c7d434020b, update_id 28ffcc1c-8e7e-437c-8370-091b081e318f. Time elapsed: 0.136 Jul 01 09:26:17.715018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.260s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:17.715418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:17.716408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:17.716738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:17.717115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:17.718343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c29c653-cee4-4435-932c-add3bd5ecd87]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:17.742638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5007e2c3-3e6a-41b6-ae55-ef9a51624ab0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:17.777337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0eb32d0-d8b3-42ce-a144-e6dee838853f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:17.853826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5bbd15d-0542-47c8-b246-47a4750c416b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 125833, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:18.500043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f798f95a-e0eb-4567-9e4e-4ad909eaf3d8]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:18.532752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d085bdf-a892-4386-980e-c33a79ca051c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:18.558709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4df5f776-05ff-4c5c-914f-904e57b277fb]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:18.588249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f4a740b-a99d-4c3e-a82f-3d50f5b721b0]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:18.589477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:18.591030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:18.591559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:18.592082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:18.615139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:18.616404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3627f2cf-7a73-4fbd-81dd-7f27956cef92]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:18.621815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5e53d35-aaca-40df-a4ff-213cd6ac4996]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:18.623818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:18.673045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:18.675021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:18.676023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:18.676606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:18.677175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:19.485059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbf93dc9-881f-4c64-aff9-51b85e7df4da]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:19.546424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3b5d68b4-c2a6-453c-80d3-1506933e2ab3]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:19.546424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:19.546424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:19.546424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:19.546424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:19.546424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:19.546424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:19.546424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:19.546424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:19.546424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:19.546424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:19.546424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:19.546424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:19.547897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b5d68b4-c2a6-453c-80d3-1506933e2ab3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:19.549054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:19.550663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:19.552695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a223c0ef-cb5a-487c-a0f5-1eeba5b95456]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:19.552695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:19.552695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:19.552960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:19.553954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:19.555077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a0be6b9-07c5-4cf6-a49a-8a9a84f20d2d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:19.556049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed44933f-d6bd-484e-bca5-77c40be50a60]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:19.557650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79117dfa-34be-4143-9376-d78f30cc0daa]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:19.594067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26c7d86a-ebdf-400c-a55e-6a55b40d69b4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 247411, 'reachable_time': 36628, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125858, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:19.604830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:19.605813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48f5a872-49d2-44f9-93d0-1f5c6d57a435]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:19.607000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:08Z", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:19.611440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:19.626912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id c2f47779-0b08-4fcc-81c9-1ac989d09e49. Wait time elapsed: 13.698 Jul 01 09:26:19.626912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id c2f47779-0b08-4fcc-81c9-1ac989d09e49. Wait time elapsed: 13.698 Jul 01 09:26:19.626912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:19.713514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.886s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:19.713720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:19.714625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:19.715030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:19.715315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:19.716469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea512e57-eb59-4ee8-a67c-a23459c172b0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:19.754170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60260ab4-2e8f-4bea-8154-929aae93ca55]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:19.802735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c180b019-317d-4bd7-a80e-4efe148c32d6]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:19.866463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a44955bd-8acd-4b5a-9ca2-c9a050d7bfaf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 125863, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:19.907900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fe73f32-f9cd-4116-8220-a253c24967fb]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:19.953979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ee37d68-6c0a-4dd0-bfbe-b13cdc2a731f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.001277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b67dbc79-e522-430c-b01f-ef96346b0bf3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.049429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5cc68a4-1487-411f-9325-f90f22605225]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.050000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:20.050353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:20.050639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:20.050890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:20.094512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:20.095049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7737b60-6c75-42bc-94e7-c513d5d23030]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.098412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1026f00-f861-4665-ad32-94913990f767]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.099538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:20.133270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:20.134055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:20.134671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:20.135563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:20.136162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:20.198220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a676aac6-44b5-449b-a6ec-bf49bdd4bd64]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.259204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[57d83f2a-1c65-4c04-a8b1-09d56f296146]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:20.259204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:20.259204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:20.259204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:20.259204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:20.259204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:20.259204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:20.259204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:20.259204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:20.259204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:20.259204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:20.259204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:20.259204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:20.260353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57d83f2a-1c65-4c04-a8b1-09d56f296146]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:20.261564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:20.264499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:20.268867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fdd369a-6827-4931-ad32-541181794c4e]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.270045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:20.270613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:20.271054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:20.271650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:20.273562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b4b4c87-ef38-4dfa-9390-c3eb53b25af5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.274761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e14a836-7f2a-43bf-a5e8-c073e9108c81]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.276519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5473375a-56cd-40e3-a4f4-3116b74593f3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.310611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[189c5574-6be6-4d1e-9195-4cc41bdaae20]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 247612, 'reachable_time': 30912, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125886, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.316039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:20.316567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[168c8072-140e-453a-8d40-894c9a931e93]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:20.317765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:14Z", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:20.320981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:20.338353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 8acfcf1d-3af9-4cc1-85e9-2d96ae9cf6de. Wait time elapsed: 22.279 Jul 01 09:26:20.338353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 8acfcf1d-3af9-4cc1-85e9-2d96ae9cf6de. Wait time elapsed: 22.279 Jul 01 09:26:20.338353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:20.883726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-762eef14-0016-4e91-ab3b-0cb734df55dc tempest-RoutersAdminTest-1421187323 tempest-RoutersAdminTest-1421187323-project-admin] Got router deleted notification for 27c7db0e-efb6-447d-810d-7741c5c7e509 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:26:20.886400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 27c7db0e-efb6-447d-810d-7741c5c7e509, action 1, priority 1, update_id 9c2ed439-1ac8-44d3-92e7-c6baa25f99ac. Wait time elapsed: 0.001 Jul 01 09:26:20.886907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 27c7db0e-efb6-447d-810d-7741c5c7e509, action 1, priority 1, update_id 9c2ed439-1ac8-44d3-92e7-c6baa25f99ac. Wait time elapsed: 0.001 Jul 01 09:26:20.887452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 27c7db0e-efb6-447d-810d-7741c5c7e509 was not found. Performing router cleanup Jul 01 09:26:20.888919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71a832c1-ffc8-4e2c-806c-07ae0cc35d2b]: (4, ['qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:20.889990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:20.890525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:20.891002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 27c7db0e-efb6-447d-810d-7741c5c7e509, update_id 9c2ed439-1ac8-44d3-92e7-c6baa25f99ac. Time elapsed: 0.005 Jul 01 09:26:21.869342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.548s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:21.869791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:21.871249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:21.871831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:21.872202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:21.874794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54797c42-6241-4734-9f19-1c7bc178e908]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:21.913387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[998a8928-eb0a-404e-a013-3d9d1d846315]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:21.951245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d6c9d08-d9d0-4681-928c-1d6e39f4f553]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.021575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[307e42dd-e973-48db-a98f-07584d2e8168]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 125901, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.061305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9996b17f-eb21-4335-b898-3936f3c08912]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.098005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a411a10-b8d2-45d9-b931-56e9bc80dfb3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.127596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c7fa825-e004-41a7-87d8-0071b22794fb]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.169153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a6ccdf3-8e79-4b12-92f1-69ec66e8e896]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.170551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:22.170975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:22.171465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:22.172126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:22.204146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:22.204650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6acedfcc-3344-4a3e-85f2-ecb242264974]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.208064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bf4bd92-08ae-4295-8375-7e426eb0743f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.209336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:22.233553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:22.234474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:22.235076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:22.235716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:22.236439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:22.274594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e15672a7-680c-4bd2-bd11-77a6d4878e80]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.343653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e0fa91b4-2ca6-483a-ad57-2bffb3421e55]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:22.343653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:22.343653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:22.343653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:22.343653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:22.343653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:22.343653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:22.343653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:22.343653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:22.343653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:22.343653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:22.343653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:22.343653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:22.345564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0fa91b4-2ca6-483a-ad57-2bffb3421e55]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:22.346689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:22.350772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:22.356375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e8787a5-cff2-4ceb-9f2e-e0378f739814]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.357340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:22.357891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:22.358319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:22.358902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:22.360791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b55a69da-2caf-488b-98bf-9694a3ca0254]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.362283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b851603-e9b7-4051-bede-d8d3b5538ead]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.363506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5329265-2847-43a6-ad49-acb5bbcc28db]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.417442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e392fe9-3477-4f17-973d-b2851e5cac47]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 247828, 'reachable_time': 27276, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 125922, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.429227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:22.429484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03cb83e0-30ef-48aa-91f2-0495f210b666]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:22.431839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:18Z", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:22.437505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:22.454785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 45974039-8826-4afd-ad22-e2d1e679b032. Wait time elapsed: 15.076 Jul 01 09:26:22.454785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 45974039-8826-4afd-ad22-e2d1e679b032. Wait time elapsed: 15.076 Jul 01 09:26:22.454785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:27.705197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 8.080s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:27.706244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:27.707353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:27.707867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:27.708312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:27.711333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c31cab1e-398d-4307-8f53-49d75c2fada7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:27.774998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fece85e7-05bb-4293-906d-2a0a9f437011]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:27.822863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5860b677-6a1d-41cc-9c74-b8bed320f445]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:27.908412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c042acc1-597a-4728-8829-2859ec9159be]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126013, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:27.959728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee2c0c0a-f34e-4199-aadc-4f98c1119ef0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.007216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6152071-f427-4ced-8fc9-09faa7ecbb4e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.054443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07e7a3c8-73da-4cb4-9cde-1cb0b53eab53]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.107880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d05a2d1-5414-4aa3-96b5-673e3c4add0a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.108722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:28.109256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:28.109686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:28.110366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:28.147505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:28.148317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eda7d03f-38e9-45f1-b4ec-8d2e1aa2ce7b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.151339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7f17d22-608a-4e08-b837-90e3100f80ad]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.152876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:28.189493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:28.190664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:28.191247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:28.191880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:28.194147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:28.238769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f0a9f8c-04eb-43cf-a663-7bb637e468bd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.312737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[36a24f9f-8037-4a68-9fce-3b461b43c636]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:28.312737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:28.312737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:28.312737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:28.312737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:28.312737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:28.312737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:28.312737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:28.312737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:28.312737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:28.312737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:28.312737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:28.312737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:28.315290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36a24f9f-8037-4a68-9fce-3b461b43c636]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:28.319872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:28.321807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:28.327467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e98091ce-cbcc-4eba-b789-c4a42bafda0d]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.328451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:28.328884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:28.329454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:28.330474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:28.331721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6659e463-5919-44f4-8402-2d7818cf5efc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.333294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbe8379a-0d13-4867-8e1e-755aa8b10439]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.336397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c358bda-1bde-4090-a62b-3d383d7e6e74]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.396648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4bfbfdf-bffa-497a-8a1a-2a061dc7273d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 248417, 'reachable_time': 39296, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126033, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.409150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:28.409722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65a0260b-114d-4c4d-a9e7-e35f9913b2c5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:28.411427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:19Z", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:28.415785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:28.435838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id fa6a959f-0fb2-42e3-96a6-c8d1e1bfdf36. Wait time elapsed: 30.800 Jul 01 09:26:28.435838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id fa6a959f-0fb2-42e3-96a6-c8d1e1bfdf36. Wait time elapsed: 30.801 Jul 01 09:26:28.435838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:29.850604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING oslo_policy.policy [None req-b230bace-72dc-4535-97c7-486617a25f77 tempest-FloatingIPNegativeTestJSON-2073546324 tempest-FloatingIPNegativeTestJSON-2073546324-project] JSON formatted policy_file support is deprecated since Victoria release. You need to use YAML format which will be default in future. You can use ``oslopolicy-convert-json-to-yaml`` tool to convert existing JSON-formatted policy file to YAML-formatted in backward compatible way: https://docs.openstack.org/oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html. Jul 01 09:26:29.852115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING oslo_policy.policy [None req-b230bace-72dc-4535-97c7-486617a25f77 tempest-FloatingIPNegativeTestJSON-2073546324 tempest-FloatingIPNegativeTestJSON-2073546324-project] JSON formatted policy_file support is deprecated since Victoria release. You need to use YAML format which will be default in future. You can use ``oslopolicy-convert-json-to-yaml`` tool to convert existing JSON-formatted policy file to YAML-formatted in backward compatible way: https://docs.openstack.org/oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html. Jul 01 09:26:29.854037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b230bace-72dc-4535-97c7-486617a25f77 tempest-FloatingIPNegativeTestJSON-2073546324 tempest-FloatingIPNegativeTestJSON-2073546324-project] Got routers updated notification :['cb920afd-d253-4fd0-8d56-68f5050b94d2'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:26:29.857577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update cb920afd-d253-4fd0-8d56-68f5050b94d2, action 3, priority 1, update_id 812eaed7-4a18-47d7-b0ba-375bfc38a257. Wait time elapsed: 0.001 Jul 01 09:26:29.858042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for cb920afd-d253-4fd0-8d56-68f5050b94d2, action 3, priority 1, update_id 812eaed7-4a18-47d7-b0ba-375bfc38a257. Wait time elapsed: 0.002 Jul 01 09:26:29.858553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:30.374089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.957s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:30.374309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:30.380101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:30.380453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:30.380740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:30.385524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3203c158-4cea-4b3a-8c0c-2be92e271062]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:30.444209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-5bfefa51-3a3c-4a76-ab25-071758e508b7 tempest-RoutersAdminTest-1423367726 tempest-RoutersAdminTest-1423367726-project] Got routers updated notification :['e09e1e70-f8ec-4445-bf77-7f05a0a19cbf'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:26:30.444926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b3ae88e-878d-4f67-a1c3-a117eeeae359]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:30.449950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e09e1e70-f8ec-4445-bf77-7f05a0a19cbf, action 3, priority 1, update_id 8f08f989-09a3-4dc4-9e86-73f6dd7ad5fd. Wait time elapsed: 0.005 Jul 01 09:26:30.450218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e09e1e70-f8ec-4445-bf77-7f05a0a19cbf, action 3, priority 1, update_id 8f08f989-09a3-4dc4-9e86-73f6dd7ad5fd. Wait time elapsed: 0.006 Jul 01 09:26:30.451168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:30.525495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a5f736b-8e79-4f08-8c4b-c0e75e498cc2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:30.640102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26b3d9ca-9a49-4eeb-835e-4cd03820e9a4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126038, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:30.695314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f4bdc27-9fc4-4cb4-b5cf-84f086c89ece]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:30.741056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.290s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:30.741401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router e09e1e70-f8ec-4445-bf77-7f05a0a19cbf was not found. Performing router cleanup Jul 01 09:26:30.743431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b544e6ee-c7bd-4d68-9985-ad2a54af3634]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:30.744024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:30.744655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:30.744922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for e09e1e70-f8ec-4445-bf77-7f05a0a19cbf, update_id 8f08f989-09a3-4dc4-9e86-73f6dd7ad5fd. Time elapsed: 0.295 Jul 01 09:26:30.745300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e031e929-f0ff-401e-89b4-456907db101c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:30.794641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6b6ad38-09f4-4c97-b4d2-289e2ca98390]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:30.833719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43216b02-0647-4f30-aca1-d56cfbbf438f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:30.835484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:30.836008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:30.836508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:30.837040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:30.887324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:30.887324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99930f63-2352-434d-8539-eb8db6e1c903]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:30.892027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ab0be29-7c4d-48cc-a9f7-1768dd5676ae]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:30.893813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:30.928669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:30.929772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:30.930318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:30.931025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:30.931650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:31.029900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5627774-7bde-4d22-82e8-c49ad1db9c4d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.119489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6ff168af-7581-448d-8d4e-b1d32bf246ae]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:31.119489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:31.119489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:31.119489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:31.119489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:31.119489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:31.119489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:31.119489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:31.119489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:31.119489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:31.119489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:31.119489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:31.119489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:31.120491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ff168af-7581-448d-8d4e-b1d32bf246ae]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:31.120785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:31.125517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:31.129797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc9e2970-4426-482a-bbc7-585cb668461e]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.131996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:31.132569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:31.132995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:31.133788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:31.135320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d754d1c-22cc-455a-a300-f2b797c50c00]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.136575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8f3cd58-4bca-44f5-a024-bdfe66a8119e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.137879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce0d0716-798f-4ed2-aa4b-695cbdc86a1f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.169638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 8.718s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:31.171093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:31.172541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:31.173094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:31.173635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:31.176670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f0b1f7f-a11d-4630-a9fe-79262302b238]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.191607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a582293-ed9b-47cf-8f83-703cf670b718]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 248690, 'reachable_time': 20880, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126059, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.196505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:31.196919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c956d69a-d0f6-4de1-b04f-e5f446fb95ff]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:31.198251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:25Z", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:31.200987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id c2f47779-0b08-4fcc-81c9-1ac989d09e49. Wait time elapsed: 25.274 Jul 01 09:26:31.220254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id c2f47779-0b08-4fcc-81c9-1ac989d09e49. Wait time elapsed: 25.274 Jul 01 09:26:31.220254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:31.225224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d4e2468-c6cb-4a42-937b-90c979602263]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.276734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7b0e441-b863-464f-80fd-7f5487c9dd19]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.422414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0054e5cb-72a2-4b4e-85a6-543248249cb6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126062, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.488490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f02d8feb-7458-4310-9913-37b06dcbd946]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.518380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f54ae5c-2d69-4932-9e20-32cff026ec7f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.570420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c261efa-49b0-4f9e-a1ea-90bcd2abd0f1]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.636930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98d5de91-d3d3-422d-ba33-7f6012474fc9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.637450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:31.637747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:31.638023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:31.638309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:31.656092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:31.656444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d76ab83-432a-47fb-b97e-0926fbbe7f9e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.659304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ed4eb1f-4f03-487a-8fe6-33961fb46e66]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.660282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:31.696446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:31.697053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:31.697454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:31.697898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:31.698226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:31.767645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61946a98-88fb-4a27-b063-dec4606e3088]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.818998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3e9b01eb-1133-49de-9f65-1fb69abdacdf]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:31.818998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:31.818998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:31.818998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:31.818998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:31.818998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:31.818998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:31.818998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:31.818998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:31.818998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:31.818998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:31.818998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:31.818998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:31.820667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e9b01eb-1133-49de-9f65-1fb69abdacdf]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:31.821813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:31.830776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:31.836367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82e15b92-739d-41a6-a86a-20c692e45db6]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.837289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:31.837782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:31.838189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:31.838688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:31.839942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6845633-5f8f-48a0-8796-c3c02075453b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.841771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d9f3b7c-639e-408a-8bb5-fe11e424424e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.843125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[119db7b3-41a5-4592-86fa-6f1bdb3011bc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.901138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c064baa1-5c55-477a-a880-3cf5b73b61ff]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 248768, 'reachable_time': 23512, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126080, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.910258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:31.910719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7e0d155-834d-4552-b71e-000ed1512962]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:31.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:22Z", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:31.915945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:31.934308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 8acfcf1d-3af9-4cc1-85e9-2d96ae9cf6de. Wait time elapsed: 33.892 Jul 01 09:26:31.934883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 8acfcf1d-3af9-4cc1-85e9-2d96ae9cf6de. Wait time elapsed: 33.893 Jul 01 09:26:31.935488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:32.369588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.511s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:32.370102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router cb920afd-d253-4fd0-8d56-68f5050b94d2 was not found. Performing router cleanup Jul 01 09:26:32.373270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72c4086c-38b7-406b-8377-2b012c16f769]: (4, ['qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:32.375086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:32.375604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:32.376068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for cb920afd-d253-4fd0-8d56-68f5050b94d2, update_id 812eaed7-4a18-47d7-b0ba-375bfc38a257. Time elapsed: 2.519 Jul 01 09:26:34.185610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.250s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:34.186896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:34.188178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:34.188842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:34.192029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:34.195006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b8b32ac-9e15-487a-bd24-3cca161d81d6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.230546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[803e3b06-56a2-46ab-8d08-df3a1fdd91cd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.268280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a9217b7-819b-4327-8187-15093fd3c588]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.336432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7751841-502e-4e7c-a6f4-e6ada99823c4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126086, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.371815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3dd32516-96ee-4009-8db9-eca1ea8e5ff6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.409433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3c774c1-3941-4e67-abcb-6265d2e7d9f8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.450903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[030a58ce-ef28-460f-82ce-a7cf679fd014]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.508460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c7bc737-9f18-4249-884c-0d7c9f50cd93]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.510398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:34.510916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:34.511423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:34.513388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:34.553182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:34.553783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff815779-8e3c-4ddc-bcf4-ff6d14c1051b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.557433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7d6f7a4-f9df-4991-a18c-97c4141679c7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.558817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:34.591728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:34.594414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:34.595101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:34.596104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:34.597159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:34.652557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56ca793e-425e-4a8e-b05e-3c99e182a025]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.723711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[02b04919-b7c1-47e4-9f3c-7a847aeb7bae]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:34.723711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:34.723711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:34.723711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:34.723711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:34.723711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:34.723711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:34.723711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:34.723711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:34.723711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:34.723711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:34.723711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:34.723711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:34.725679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02b04919-b7c1-47e4-9f3c-7a847aeb7bae]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:34.727607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:34.731266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:34.734331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c2be66f-5f0e-49fb-a61c-488203ae0d0f]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.735237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:34.735714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:34.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:34.736739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:34.737951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1b43f67-2131-4552-8cc0-ae170b29bbf2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.739174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8be8a45a-ef68-4c44-a38b-667c3e66fde1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.740532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f6191fe-9b57-4c18-b668-b6735cca1d59]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.826382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22788d9d-1bed-4508-8393-ad645a404369]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 249059, 'reachable_time': 28956, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126107, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.835323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:34.835925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a649fac-0926-4632-8980-6ae8824cb885]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:34.837280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:22Z", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:34.842428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:34.863503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 45974039-8826-4afd-ad22-e2d1e679b032. Wait time elapsed: 27.481 Jul 01 09:26:34.863503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 45974039-8826-4afd-ad22-e2d1e679b032. Wait time elapsed: 27.485 Jul 01 09:26:34.863503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:37.168506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c6da7081-e770-445e-8bb1-2eb3de071172 tempest-RoutersAdminTest-1423367726 tempest-RoutersAdminTest-1423367726-project] Got router deleted notification for e09e1e70-f8ec-4445-bf77-7f05a0a19cbf {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:26:37.169394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e09e1e70-f8ec-4445-bf77-7f05a0a19cbf, action 1, priority 1, update_id e1bf6917-b454-4698-ac17-be14b9bbede6. Wait time elapsed: 0.001 Jul 01 09:26:37.169644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e09e1e70-f8ec-4445-bf77-7f05a0a19cbf, action 1, priority 1, update_id e1bf6917-b454-4698-ac17-be14b9bbede6. Wait time elapsed: 0.001 Jul 01 09:26:37.170085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router e09e1e70-f8ec-4445-bf77-7f05a0a19cbf was not found. Performing router cleanup Jul 01 09:26:37.171184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[966cb9c1-1bca-4b9d-8c25-0306b3dd2068]: (4, ['qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.172185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:37.172622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:37.173024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for e09e1e70-f8ec-4445-bf77-7f05a0a19cbf, update_id e1bf6917-b454-4698-ac17-be14b9bbede6. Time elapsed: 0.004 Jul 01 09:26:37.369447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 6.169s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:37.369772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:37.370752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:37.371077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:37.371375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:37.372895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce0f2253-bdf1-43f5-a4ad-e6030e708db2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.408847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff85e45e-e0da-4325-aad4-03c46c130c29]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.470043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae9b2a8e-e311-47fb-ada7-be32b6ffcedf]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.551745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b10e018-4198-462c-96f5-6f651e8d9532]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126111, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.595964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2a300dd-4cd9-4037-b10a-6ed89d949ae6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.633533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa9c768d-72c3-4e44-868b-9fcad5e6728c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.670875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[893db058-792c-4d90-a283-7314eb9df52d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.706868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab70eba0-f4b7-4c9f-90b6-3f000649c816]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.707455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:37.707732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:37.707994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:37.708262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:37.742599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:37.743130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cae177e2-d608-440f-91ba-cdd131e1bcd6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.746018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d91c1f9-f72e-4746-aa12-2d40254dd6a7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.747283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:37.775513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:37.776445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:37.776977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:37.777519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:37.778048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:37.817895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13afaa53-7812-478c-8142-215b5bf2b641]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.867002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f2780180-d9f4-4694-9653-c92c3403e9b0]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:37.867002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:37.867002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:37.867002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:37.867002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:37.867002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:37.867002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:37.867002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:37.867002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:37.867002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:37.867002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:37.867002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:37.867002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:37.868558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2780180-d9f4-4694-9653-c92c3403e9b0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:37.869676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:37.872791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:37.876616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d510b2a-0096-4cae-b7fa-5cd8122f5b93]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.877553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:37.878045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:37.878453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:37.878964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:37.880364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04713f89-8553-4f77-ab64-11aee4f4d928]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.882357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd9d7146-cebf-424f-ae65-213f188b77de]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.883490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fcf2025-c897-435c-a213-8482dde467b8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.920032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0cf3baa-4cae-4bc0-887e-d3a31b83514e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 249381, 'reachable_time': 34596, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126131, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.927051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:37.927656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1541457-4d17-4f49-abb1-535273f7c0f1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:37.928913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:25Z", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:37.932849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:37.948328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id fa6a959f-0fb2-42e3-96a6-c8d1e1bfdf36. Wait time elapsed: 40.328 Jul 01 09:26:37.948328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id fa6a959f-0fb2-42e3-96a6-c8d1e1bfdf36. Wait time elapsed: 40.329 Jul 01 09:26:37.948328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:39.255263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.310s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:39.257164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:39.258838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:39.259248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:39.259688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:39.262477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d4b7a15-59ec-450a-aeb3-0d3acabd17c1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.294916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b51f291-626d-4092-aa97-833cf26e913a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.337300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6816afe8-da9d-4c66-a616-6dbe1e3472ac]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.416618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c9eb35f-3654-44df-bb95-fbf1dfd00841]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126134, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.440022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cd32164-0e6a-4b9d-b10a-dfc90523c16b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.474204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a97f5c46-f96d-4a4a-a4eb-8081282d77de]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.513334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82727ee5-e508-4cd0-98a5-edbd7667da98]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.552621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfb1564c-c380-4399-a71b-9f57e35081de]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.554378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:39.554842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:39.555282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:39.555821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:39.589853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:39.590616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7b749a6-7dd2-460d-901b-1ef003d9ec65]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.593965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04651a7a-958b-49f0-b66f-129cf5fa9ba3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.595981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:39.634526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:39.635481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:39.635929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:39.636373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:39.637052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:39.693551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e69d0638-f80f-421b-8850-06ec6ee3e9bf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.750682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4cfc34d4-f726-4a13-a4b3-aad3258c2d59]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:39.750682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:39.750682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:39.750682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:39.750682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:39.750682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:39.750682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:39.750682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:39.750682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:39.750682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:39.750682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:39.750682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:39.750682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:39.752121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cfc34d4-f726-4a13-a4b3-aad3258c2d59]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:39.753216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:39.756267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:39.760479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7e7f686-d2d4-4710-9c14-e08f955b0ffd]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.761391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:39.761866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:39.762266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:39.763948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:39.767277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba694c13-8ec3-4e0e-a996-ad3bdc795050]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.768490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc73e0b9-2f2a-4ec7-bdee-9e085cb9e770]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.770249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[624fd8f7-5f01-4441-81a9-a14233d6461e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.812523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eef704fa-138c-4799-9b2f-08a60331004b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 249567, 'reachable_time': 18648, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126154, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.821824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:39.822073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8efa658-74fd-4c5b-88cf-2207bf9e1f7a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:39.823298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:35Z", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:39.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:39.854072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3 Jul 01 09:26:39.854072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id c2f47779-0b08-4fcc-81c9-1ac989d09e49. Wait time elapsed: 33.901 Jul 01 09:26:39.854072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id c2f47779-0b08-4fcc-81c9-1ac989d09e49. Wait time elapsed: 33.902 Jul 01 09:26:39.854072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:41.700452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 6.841s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:41.701981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:41.703585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:41.705328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:41.728128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.023s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:41.739328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb6822dd-21e7-4d03-a87a-9f675d4e89d8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:41.796084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3894dce6-be4f-4308-afcb-913e5712dd0b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:41.848940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f4ce97c-cd3a-4c0a-bd85-fa1dea29ac5b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:41.915134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c4c8aee-fcc2-4d54-a5ba-968504cdc7bf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126223, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:41.957005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45f5b220-91ca-4b19-9485-82d43361f068]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.002969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[003f1913-190a-4a94-b8e8-922e4e98829d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.043802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd27d7e1-02ec-4bd9-9453-4b23b43793e4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.077452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a586eec0-769b-470b-80ea-0a34c1ea1cae]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.078357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:42.078790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:42.079281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:42.079794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:42.112759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:42.114425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87922dd6-6772-4a7c-9b8a-54fd33770265]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.121418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d451e84-2e50-44c3-b4e4-81621185bf57]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.123694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:42.165680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:42.166842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:42.167408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:42.167985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:42.169224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:42.218413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43b21005-622f-4d99-8ab7-8ff8c88422b6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d3ac88ee-b3f9-41f2-b919-cdce849ffcb1]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:42.297406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3ac88ee-b3f9-41f2-b919-cdce849ffcb1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:42.299635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:42.301904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a504f184-b9c9-40c2-ae64-9be83b3368ef]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.305075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:42.305075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:42.305075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:42.305075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:42.307176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8371847-ebd0-4b15-b602-5ac783897c9e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.308455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[534c4a6f-e0f3-42cb-a403-6e8d94f92a46]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.311347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[501e0b2f-0fbc-4d7d-a48b-824d5b8773b9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.364215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55146ca9-3b1b-41da-9a9b-b66698789097]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 249817, 'reachable_time': 28732, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126243, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.393765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:42.394738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[723b1ab8-f723-4305-b5e6-56a7f6d03fee]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:42.397928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:37Z", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:42.404828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:42.435758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 8acfcf1d-3af9-4cc1-85e9-2d96ae9cf6de. Wait time elapsed: 44.363 Jul 01 09:26:42.435758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 8acfcf1d-3af9-4cc1-85e9-2d96ae9cf6de. Wait time elapsed: 44.365 Jul 01 09:26:42.435758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:43.535615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c1aeac89-e21e-4572-a316-6ccfc1649595 tempest-RoutersAdminTest-1421187323 tempest-RoutersAdminTest-1421187323-project-admin] Got routers updated notification :['7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:26:43.541112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06, action 3, priority 1, update_id f6564dc8-dda1-44fc-b08d-9f3f49ee76af. Wait time elapsed: 0.001 Jul 01 09:26:43.541595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06, action 3, priority 1, update_id f6564dc8-dda1-44fc-b08d-9f3f49ee76af. Wait time elapsed: 0.002 Jul 01 09:26:43.542276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:43.759857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.345s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:43.760385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:43.761605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:43.762118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:43.763119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:43.765273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56ca35fb-2bed-4ed8-875d-7cfc852cac9c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:43.780611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42016241-46ac-4637-8b01-4ef05024a500]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:43.823885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d36dbbf0-5e9c-4af8-9590-3e3169aadc30]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:43.883660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efc38bf7-922d-48b4-b87b-3e84c49ae14e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126246, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:43.929682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ca03edb-e8e5-46e6-b32b-cac569268c65]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:43.967336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f0c0fac-af2d-4ea3-9e4e-9e77b083e2e3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:44.009565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2dc9b7ec-7fdd-4cc2-bdb7-0a5d2145be47]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:44.045269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48ced4f9-1266-48e9-8a4d-8a5db1e94ad9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:44.047178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:44.048023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:44.048574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:44.049529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:44.083001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:44.083266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a9c976b-c324-414b-88ee-efef98103da4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:44.088995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1cc9d112-f48d-4cf7-b99b-53309f6e30e4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:44.090213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:44.129934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:44.136066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:44.136825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:44.137362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:44.138658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:44.218223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff369733-8c11-4898-a432-077de0ab1530]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:44.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[dd4ce9b0-8277-47b5-a9c5-7c5bfc3b5c63]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:44.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:44.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:44.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:44.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:44.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:44.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:44.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:44.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:44.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:44.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:44.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:44.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd4ce9b0-8277-47b5-a9c5-7c5bfc3b5c63]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:44.272204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:44.274339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:44.276788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bb8f6ce-adaf-4b70-9955-15319f1154e1]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:44.276788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:44.276788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:44.276788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:44.276788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:44.277829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5ca16f8-66ad-400f-bd82-c374f90170f6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:44.281415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc0206f3-9e11-4a87-b65f-af6a9abe5dbe]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:44.282519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12bdc007-2a01-414b-9539-4528a359017f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:44.317430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1970a90d-0f1b-4f44-855b-9bb3c5d25f89]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 250014, 'reachable_time': 32780, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126265, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:44.326711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:44.327091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07735b1c-bcbd-46d3-85f1-7047ef957ab8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:44.329219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:37Z", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:44.332435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3 Jul 01 09:26:44.348201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 45974039-8826-4afd-ad22-e2d1e679b032. Wait time elapsed: 36.956 Jul 01 09:26:44.348201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 45974039-8826-4afd-ad22-e2d1e679b032. Wait time elapsed: 36.957 Jul 01 09:26:44.348201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:45.160389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.829s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:45.164227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:45.165518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:45.165989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:45.166354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:45.167700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba778f4a-a90c-4685-bb35-c8c404291925]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.186928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55145419-60d6-4f99-903f-b9aae2b684d5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.223654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e16dc140-7f8a-451c-98f4-d2d23b5087ad]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.278598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd7c7b7c-888e-4433-8193-2cd61bd63d2c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126270, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.309511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e95ba02-532e-4ec8-925c-186f3b1562e8]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.333182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eff0a0ba-6c52-46d7-9aad-ecae01d867ed]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.365066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2d8dead-5fe2-461f-a5f8-30865526d0b5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.378099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.550s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:45.378371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:45.381839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:45.382303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:45.382579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:45.385954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b756f1a4-f1f6-4344-b570-3ef01c7ff899]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.447187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be55ce38-ebe8-4ed6-9022-651458f2a8bd]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.448002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:45.448437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:45.448830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:45.449388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:45.456764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b9b3720-6a85-4306-9731-b3c913beb7c2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.485032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:45.485285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49e31d18-9983-4086-94f9-dce180a8b556]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.489380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5dd98ecf-1d8c-4019-bc89-8034fdbe16a7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.490944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:45.500606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f3896e2-f29f-47ad-ade0-6fcf41f4bf44]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.528009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:45.528978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:45.529819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:45.530357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:45.531653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:45.589017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5b95681-df05-407d-a52e-976baac50941]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.597576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4698ab7-a43e-438a-ae02-beb3ff3ba410]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126281, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.637102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8849e4bc-a7eb-4979-b055-5348d0f1c2f1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.647491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e8b50811-dcf9-433d-a226-40d5a5aac58f]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:45.647491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:45.647491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:45.647491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:45.647491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:45.647491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:45.647491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:45.647491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:45.647491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:45.647491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:45.647491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:45.647491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:45.647491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:45.648531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8b50811-dcf9-433d-a226-40d5a5aac58f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:45.649131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:45.650982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ee1de6e-e134-4339-bf25-c77b8135a81a]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.653512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:45.653512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:45.653512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:45.653512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:45.655571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[463b3d61-3999-48bb-baf5-d8aa1c54ccf8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.656640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50277313-1dfe-4ada-b87a-1438de1fa0f4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.657669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ca2ecf8-280d-4422-8c62-07577c797f47]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.690054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcab0cb8-8817-4c2e-b154-6ff214d42956]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.697138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ebc40a4e-fdf5-4a9d-9218-578d176681d6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 250154, 'reachable_time': 44108, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126296, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.701809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:45.702085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ead442e7-7487-4c35-9a4d-51c1ec12fee8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:45.703085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:44Z", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:45.706484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 45974039-8826-4afd-ad22-e2d1e679b032. Wait time elapsed: 38.330 Jul 01 09:26:45.723462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 45974039-8826-4afd-ad22-e2d1e679b032. Wait time elapsed: 38.330 Jul 01 09:26:45.723462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:45.742430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f081377-79d9-4bc5-ae02-4be98e2c9349]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.773911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67342fc7-8791-4d43-a654-1af34f52abe6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.774487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:45.774732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:45.775003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:45.775274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:45.810672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:45.810904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0db057db-ad47-49bc-b34c-2127b1c9b330]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.813581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23b8ca21-754e-4d07-a5ec-6d6707c8a31d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.814511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:45.840162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:45.840754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:45.841138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:45.841514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:45.841887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:45.881248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e38b7577-5b97-47f9-acc2-643aa8875f65]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.939715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8188c3a3-32c7-4625-8a72-76fee9f6cb82]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:45.939715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:45.939715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:45.939715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:45.939715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:45.939715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:45.939715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:45.939715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:45.939715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:45.939715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:45.939715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:45.939715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:45.939715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:45.941055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8188c3a3-32c7-4625-8a72-76fee9f6cb82]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:45.942135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:45.944887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:45.950745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c0dc5dd-95e9-4abe-845b-342d02e5d895]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.951677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:45.952235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:45.952656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:45.953235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:45.954561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20c69b3a-d78a-4852-95d2-3c66c64d015a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.955755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e7dc5e0-0801-4127-9725-ce18cae751c9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.956964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6e1ff51-a4a8-4c38-9b24-23529b6652e0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:45.996863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[111eeccb-4c6b-46f3-8ff9-ea3e80302438]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 250185, 'reachable_time': 27280, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126311, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:46.000152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:46.000684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db18086a-8c7f-4a7e-b714-8498a6857eaf]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:46.002708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:39Z", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:46.007174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:46.019203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id c2f47779-0b08-4fcc-81c9-1ac989d09e49. Wait time elapsed: 40.090 Jul 01 09:26:46.019203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id c2f47779-0b08-4fcc-81c9-1ac989d09e49. Wait time elapsed: 40.090 Jul 01 09:26:46.019203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:47.478377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.461s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:47.479514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:47.482748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:47.483312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:47.484623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:47.486275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f126859a-84f7-438a-8d96-6926a2e5678b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.517827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8eb6052-0eae-488b-8c5f-b1a94ac23898]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.562532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f3620c4-014d-4c57-a375-7a1744246dba]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.623220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9254513f-a277-4149-aad7-01d44e118d3c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126314, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.668011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9538312f-b9e7-4e8e-beaa-ab4281de1cc3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.707265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[139cb88c-8d39-4f71-9171-7c37d77baeff]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.750022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52675d24-a025-446a-b9cd-974f77e61940]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.795849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6d202a1-23ab-401e-bfcb-c631137c1408]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.797048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:47.797616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:47.798277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:47.798984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:47.824393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:47.825319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fbe89f8-3a4a-447f-814e-4fef239fc867]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.828979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[375dc195-8f44-42d8-9a68-66376e60b3f6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.830327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:47.867515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:47.868459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:47.869067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:47.869908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:47.870494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:47.937877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0b40fc7-333c-4d12-9379-eea4ce4828bd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.975113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e14f283e-9324-4ee9-8c8c-8166b1a00679]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:47.975113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:47.975113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:47.975113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:47.975113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:47.975113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:47.975113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:47.975113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:47.975113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:47.975113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:47.975113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:47.975113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:47.975113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:47.976117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e14f283e-9324-4ee9-8c8c-8166b1a00679]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:47.977024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:47.978575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:47.980563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f21aed49-3142-4f7f-bb3c-c7638ed08243]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.980563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:47.980717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:47.981493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:47.981930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:47.983035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b253712d-c0e3-4578-8f96-887780faa2ee]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.984377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da20a2e1-16a0-43eb-b8fb-16a174d2a74d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:47.985822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34b5a1bb-2c9f-4ca2-9d28-a469da8a9a54]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:48.013072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85eff91a-e003-42de-96d6-4f4d806973c3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 250388, 'reachable_time': 16436, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126333, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:48.020079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:48.020689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca79f3ae-e608-4109-bbb0-c94e1fc2e401]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:48.021949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:39Z", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:48.026945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:48.041927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id c2f47779-0b08-4fcc-81c9-1ac989d09e49. Wait time elapsed: 42.113 Jul 01 09:26:48.041927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id c2f47779-0b08-4fcc-81c9-1ac989d09e49. Wait time elapsed: 42.114 Jul 01 09:26:48.041927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:49.064968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.360s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:49.065852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:49.066932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:49.067427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:49.067883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:49.069631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e6dc8ce-1793-44cc-9cb6-7c4ab5074a72]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.098148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b20f0e3-a79e-4c64-a238-ed6aa3ced888]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.137210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca3f8268-2df6-4b91-8b74-47ccacb0f0c8]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.167843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bf29a1f-430e-4f34-a6e8-29dc2cc61992]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126351, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.190020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8af63700-02ba-4cae-8c8a-e5168e49c564]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.218239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de1f0bc4-8ced-40b7-a51e-42a3aea7733f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.254602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7dc56c83-7621-4f5b-a724-19d89d28636f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.320493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c85a1ad5-0257-4853-8e2f-9ab4dc2b9ed1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.322010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:49.322493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:49.322948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:49.323556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:49.343724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:49.344233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[080d17af-091a-47aa-98ae-9233888572b7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.347047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7398868-8903-4ce3-97ef-7fb081c8847b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.348191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:49.388479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:49.397010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:49.397698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:49.399519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:49.400280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:49.457199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06d21536-c03d-4bb1-884b-1bbf40d8160c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.493916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[83ca7b30-044b-4a9b-b0aa-047543e7e157]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:49.493916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:49.493916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:49.493916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:49.493916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:49.493916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:49.493916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:49.493916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:49.493916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:49.493916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:49.493916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:49.493916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:49.493916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:49.495124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83ca7b30-044b-4a9b-b0aa-047543e7e157]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:49.495613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:49.497599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:49.500071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe61cb25-45a6-4816-bde3-b8cb63271b65]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.500071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:49.500071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:49.500071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:49.500071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:49.501959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec3606c4-a69d-4160-8a4a-4dd753fdb5c3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.502925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56746228-ae59-41a3-8a8d-aa0b8e073753]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.504295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[857eb123-eddd-4cac-a00a-9fe36e13b3e4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.540410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8edf6cf4-408d-4371-8ff2-b73e29f1ddb0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 250543, 'reachable_time': 26696, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126369, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.549756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:49.550243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[424f3722-0555-4ef9-b553-320ae35a7beb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:26:49.552047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:44Z", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:49.557283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:49.574676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3 Jul 01 09:26:51.094299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-9018c381-1c11-46f7-a6c7-4a0bae2d50da tempest-RoutersAdminTest-1421187323 tempest-RoutersAdminTest-1421187323-project-admin] Got routers updated notification :['ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:26:51.098263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74, action 3, priority 1, update_id 40c1cac3-2f90-4bad-88c5-a64787eaf4c6. Wait time elapsed: 0.002 Jul 01 09:26:51.098953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74, action 3, priority 1, update_id 40c1cac3-2f90-4bad-88c5-a64787eaf4c6. Wait time elapsed: 0.003 Jul 01 09:26:51.099708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:52.188519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.148s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:52.189139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:52.189704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:52.190031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:52.190316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:52.192075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3da1e376-36e5-4ab2-acbc-036c7834223f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.231730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6fec99f-ecfc-45e4-9b5f-896d93c8c675]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.261998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 8.719s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:52.262601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:52.264549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:52.266265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:52.266904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:52.269067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66182500-b914-4f83-ab8a-67ea1ac080f3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.279517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db713424-faf1-4228-aa85-41c7e3cb1d0f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.300767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[805664d0-d01a-40fe-a788-dd888e7b39d6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.330627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d3ce41e-2f07-4b01-b200-64d9adf5a0b1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.346167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8955498a-fc05-4fa4-abe8-6844aef7a0f5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126375, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.383315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de461b4f-b451-4a56-b46a-cd5249ed4152]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126378, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.397678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95b252e9-1752-44d1-83e0-8158d2a98ced]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.428085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31843778-ea2d-4c11-9dea-41e06e88c406]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.437321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8eee302e-4f12-4b97-8e6b-f90b580167b0]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.467600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-9919dc3c-5ae6-4f6f-8e55-5b294ef3e71c tempest-FloatingIPNegativeTestJSON-2073546324 tempest-FloatingIPNegativeTestJSON-2073546324-project] Got routers updated notification :['cb920afd-d253-4fd0-8d56-68f5050b94d2'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:26:52.469022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update cb920afd-d253-4fd0-8d56-68f5050b94d2, action 3, priority 1, update_id 927063d6-4997-4041-8d79-9797de9e3a96. Wait time elapsed: 0.001 Jul 01 09:26:52.469493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for cb920afd-d253-4fd0-8d56-68f5050b94d2, action 3, priority 1, update_id 927063d6-4997-4041-8d79-9797de9e3a96. Wait time elapsed: 0.001 Jul 01 09:26:52.470540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:52.474840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6192595c-369e-4c21-ae17-1637f5205480]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.475496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62d50f7e-26c2-4aa8-aab8-a3748fdd5b88]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.499811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6052d161-a4e1-49cf-bb48-69ae84fbe75c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.500569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:52.500856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:52.501124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:52.501403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:26:52.513028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86c9c670-aa42-4244-b661-9927993d3967]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.529064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:52.529396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9155b4f9-28a1-4956-b9b0-495ba49977de]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.533424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5749a66a-8f3a-428f-9f89-503570074a7c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.534666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:52.566423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17e23726-f669-4a29-8be0-4c4970508909]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.570406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:52.571395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:52.572132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:52.572721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:52.573346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:52.589481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:52.593522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.004s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:52.607051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:52.641553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23836536-e708-4510-a1eb-3b9b8eb91692]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.642450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03ead248-dc59-40fb-9129-b22c069d10e1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 250864, 'reachable_time': 33056, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126399, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.643173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:52.643473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:52.643926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:52.644190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:26:52.644529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:52.644789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:52.645080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '6f2b0b43-44c3-4f4a-8c14-3638c1c80ebb', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:dc:e5:61', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'device_owner': 'network:router_gateway', 'standard_attr_id': 591, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.9', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:26:39Z', 'updated_at': '2022-07-01T09:26:50Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-6f2b0b43-44), ns(qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:26:52.678046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:52.678468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8bf96da-adab-49d9-81cb-6544f76be891]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.681080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4dbb721d-8b5d-41a5-9d39-484da7d8fdcf]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.682023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-6f2b0b43-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:52.682346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:26:52.683253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-6f2b0b43-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:52.684321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-6f2b0b43-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:52.685151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-6f2b0b43-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:52.685951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-6f2b0b43-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:52.686732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-6f2b0b43-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6f2b0b43-44c3-4f4a-8c14-3638c1c80ebb', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:dc:e5:61'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:52.745096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98c87aa3-0068-442d-aa04-4bdc922ac4e9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b6ef11d1-9b53-41c9-a2b6-a22ec0cfba9a]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:52.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:52.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:52.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:52.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:52.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:52.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:52.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:52.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:52.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:52.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:52.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:52.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:52.753681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6ef11d1-9b53-41c9-a2b6-a22ec0cfba9a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:52.757496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:52.760421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:52.766448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[640caa72-c6df-4aee-aa0c-d1bc277c3ba6]: (4, ['qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.769443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:52.771488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:52.771919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:52.773244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:52.775780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9062cea-61bb-4194-bc6c-34ef72443d03]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.778213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8cb1438d-dcf0-41b5-b4a9-2aa4287f55e6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.780539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbc6de0f-df5d-4635-8118-f23cad38a89e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.822208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[81e33fbb-927f-4d8d-b45b-9903c0043910]: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:52.822208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:52.822208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:52.822208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:52.822208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:52.822208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:52.822208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:52.822208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:52.822208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:52.822208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:52.822208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:52.822208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:52.822208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:52.826079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81e33fbb-927f-4d8d-b45b-9903c0043910]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.827073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:52.829740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:26:52.830508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f665af5-b205-4630-a0b0-3352e6e765cc]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 250860, 'reachable_time': 29528, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126413, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.835510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:52.836493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1889f50b-4b9f-42cf-87da-3a096589e259]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:26:52.838288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:49Z", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:26:52.842362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:52.856246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3 Jul 01 09:26:52.869990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea53a6f5-c221-4225-9271-85d622bd029a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.081689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ad26a99-8d0a-4dce-b651-5d9deec9b9c6]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.121070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0229dbb2-7bae-46a1-ae53-347d90ff49e3]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.293598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d292bff7-9148-4e8b-93e5-6c8e569c2ba1]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.294318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:26:53.294559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:53.295057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:53.295584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" released by "process_external" :: held 0.651s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:53.296705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:53.300175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:53.306404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac7cbf50-9497-4619-bd72-8cbf27f640f1]: (4, ['qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.307393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:53.307933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:53.308340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:53.308843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:53.310168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ebbc82c-df57-48a1-821c-9e0a4c9766b2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.312001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4583e182-743f-49a7-98dd-cf4754e7629f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.313244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[600e0e0a-fea7-42cd-b845-1d84db9d0724]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.354187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29b58d28-8b96-4af9-8b2e-6d2c24d57dd7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 250864, 'reachable_time': 33056, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126418, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.357182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:53.357657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b19a9f6-8dd0-4f2b-b71b-143b33f028bc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:53.359138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-snat-router-969169340", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f6303cc2036a435290b0447468eda290", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": false Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f2b0b43-44c3-4f4a-8c14-3638c1c80ebb", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:37Z", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:40Z", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f6303cc2036a435290b0447468eda290", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f2b0b43-44c3-4f4a-8c14-3638c1c80ebb", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dc:e5:61", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 591, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:39Z", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:50Z", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": false, Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:26:53.363863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:53.373307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06, action 3, priority 2, update_id f6564dc8-dda1-44fc-b08d-9f3f49ee76af. Wait time elapsed: 9.832 Jul 01 09:26:53.373307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06, action 3, priority 2, update_id f6564dc8-dda1-44fc-b08d-9f3f49ee76af. Wait time elapsed: 9.832 Jul 01 09:26:53.373307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:53.412888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.313s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:53.413801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:53.415861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:53.416375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:53.416821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:53.418197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff828220-3cdd-4171-a113-0145fe146628]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.441085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79a6de00-811e-424b-9877-01b0e26300b4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.472794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd89bccf-4ae7-4f02-80fc-ae436d10844a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.515749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[383dd778-d087-41f6-9c30-3d540d61b6c5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126421, 'error': None, 'target': 'qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.554619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd660e42-bbb1-44b8-977b-7417b389b2e5]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.593027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2a03496-8180-4845-937e-f55f9329f5b1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.627724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.157s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:53.628240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router cb920afd-d253-4fd0-8d56-68f5050b94d2 was not found. Performing router cleanup Jul 01 09:26:53.629412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[806cfef3-08ab-4428-928c-b0a30caa0688]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.631968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a00ca160-8973-4689-9f4c-340601d06b42]: (4, ['qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.633716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:53.634648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:53.635729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for cb920afd-d253-4fd0-8d56-68f5050b94d2, update_id 927063d6-4997-4041-8d79-9797de9e3a96. Time elapsed: 1.167 Jul 01 09:26:53.667798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1af2ba10-14ca-41c0-ae8b-612f4023ed8f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.669187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:53.669725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:53.670202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:53.702056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fbc75ce-ff70-486c-b170-94202ac22440]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 250978, 'reachable_time': 28708, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126429, 'error': None, 'target': 'qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.705900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:53.706284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:53.707002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:53.707864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:26:53.709438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:53.710039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:53.710563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'b0ef3088-717a-4d1b-b4d2-c40a12fb11f5', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:09:9e:90', 'admin_state_up': True, 'status': 'DOWN', 'device_id': 'ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'device_owner': 'network:router_gateway', 'standard_attr_id': 618, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.49', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:26:46Z', 'updated_at': '2022-07-01T09:26:52Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-b0ef3088-71), ns(qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:26:53.748222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-b0ef3088-71 not found in namespace qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:53.749819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d13345b6-376c-4fe3-b95a-5a2dec935e97]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.753612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd443dfd-2eef-48a1-b6b9-0db39e4d9feb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.755999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-b0ef3088-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:53.756642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:26:53.757664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-b0ef3088-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:53.759781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-b0ef3088-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:53.760478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-b0ef3088-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:53.761112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-b0ef3088-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:53.762006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-b0ef3088-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0ef3088-717a-4d1b-b4d2-c40a12fb11f5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:09:9e:90'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:53.844042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3e21e9f-19f8-40e7-8820-46e4aeba626b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.871325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[17f08e79-2e69-4bb6-b893-2e0d0953863d]: Network interface qg-b0ef3088-71 not found in namespace qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:53.871325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:53.871325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:53.871325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:53.871325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:53.871325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:53.871325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:53.871325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:53.871325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:53.871325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:53.871325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:53.871325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:53.871325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b0ef3088-71 not found in namespace qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74. Jul 01 09:26:53.873585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17f08e79-2e69-4bb6-b893-2e0d0953863d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-b0ef3088-71 not found in namespace qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:53.874608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:53.875459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:26:53.923438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c1834b7-b160-49bb-8b5e-4e9883a10915]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.131304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d744901-6145-40de-ae83-58847d4cd834]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.188725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[caba3c86-2c53-4fef-9b2f-96213a8ab00c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.405748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a50d342-f94b-409b-93c6-f43e79ec5193]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.406385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:26:54.407121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:54.407719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:54.408194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" released by "process_external" :: held 0.700s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-b0ef3088-71 not found in namespace qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b0ef3088-71 not found in namespace qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74. Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b0ef3088-71 not found in namespace qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74. Jul 01 09:26:54.409531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b0ef3088-71 not found in namespace qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74. Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b0ef3088-71 not found in namespace qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74. Jul 01 09:26:54.417776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:54.424289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[254895d6-1dee-4c5f-9a0d-3eb43179ad7f]: (4, ['qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.425918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:54.429824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:54.430400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:54.431012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:54.433206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5e0b5d9-c124-40e7-ac36-79456ff72fc2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.441401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a929136-42f6-461d-a2fa-0b0003694ca1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.449826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09a6abf5-366c-4d42-855b-eb5b95f241f5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.493271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbf66fba-e9fe-456e-a7be-b648560d8649]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 250978, 'reachable_time': 28708, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126446, 'error': None, 'target': 'qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.495526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:54.496009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9a82aeb-02e1-4889-a1a7-0228c28d8e37]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b0ef3088-71 not found in namespace qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74. Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b0ef3088-71 not found in namespace qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74. Jul 01 09:26:54.498306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-snat-router-969169340", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f6303cc2036a435290b0447468eda290", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.49" Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b0ef3088-717a-4d1b-b4d2-c40a12fb11f5", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:44Z", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:49Z", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f6303cc2036a435290b0447468eda290", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0ef3088-717a-4d1b-b4d2-c40a12fb11f5", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:09:9e:90", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 618, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.49", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:46Z", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:52Z", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:26:54.502827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:54.511569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74, action 3, priority 2, update_id 40c1cac3-2f90-4bad-88c5-a64787eaf4c6. Wait time elapsed: 3.414 Jul 01 09:26:54.511569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74, action 3, priority 2, update_id 40c1cac3-2f90-4bad-88c5-a64787eaf4c6. Wait time elapsed: 3.414 Jul 01 09:26:54.511569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:54.800427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-12739dfa-95dc-43c9-b7fb-26ac0f54d5d4 tempest-FloatingIPNegativeTestJSON-2073546324 tempest-FloatingIPNegativeTestJSON-2073546324-project] Got router deleted notification for cb920afd-d253-4fd0-8d56-68f5050b94d2 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:26:54.801567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update cb920afd-d253-4fd0-8d56-68f5050b94d2, action 1, priority 1, update_id 474afea6-38bf-4ca1-89a4-a987f6abcaed. Wait time elapsed: 0.001 Jul 01 09:26:54.802463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for cb920afd-d253-4fd0-8d56-68f5050b94d2, action 1, priority 1, update_id 474afea6-38bf-4ca1-89a4-a987f6abcaed. Wait time elapsed: 0.001 Jul 01 09:26:54.802935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router cb920afd-d253-4fd0-8d56-68f5050b94d2 was not found. Performing router cleanup Jul 01 09:26:54.804370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30512a6e-aca2-4fe2-9e86-72106424f4f3]: (4, ['qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.804836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:54.805250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:54.805498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for cb920afd-d253-4fd0-8d56-68f5050b94d2, update_id 474afea6-38bf-4ca1-89a4-a987f6abcaed. Time elapsed: 0.004 Jul 01 09:26:54.818744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.446s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:54.819173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:54.820422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:54.820889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:54.821338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:54.823028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0313ade9-69f7-43c5-9482-18715ac991ed]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.858904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd24059f-c9b6-4021-a278-a4bffd641465]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.884314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29c56040-e6e6-4297-86d4-cc8d4202ec48]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.947300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30a5908b-0878-4594-ac67-a07531ca667b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126451, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:54.997457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98d7efaf-f68a-4f44-856b-d84f653cc97a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:55.044870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07a75163-7045-4a27-8fa6-cf70fa81dad4]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:55.069439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:26:55.089571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0655794-62c8-4b2b-90bd-8ed246cf36ce]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:55.150661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0df523d1-974a-44c2-af91-9a593e3b3680]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:55.152333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:55.153736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:55.154237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:55.178231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1b3a307-3758-4327-9112-b572afc19e36]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251120, 'reachable_time': 29384, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126457, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:55.183097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:55.183754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.004s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:55.188129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:55.195855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" acquired by "process_external" :: waited 0.008s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:26:55.198198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:55.198742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:55.199333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '6f2b0b43-44c3-4f4a-8c14-3638c1c80ebb', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:dc:e5:61', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'device_owner': 'network:router_gateway', 'standard_attr_id': 591, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.9', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:26:39Z', 'updated_at': '2022-07-01T09:26:50Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-6f2b0b43-44), ns(qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:26:55.264743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:55.265840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[333aeb6d-0bba-4914-ba59-0d0f4749d2c4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:55.275019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d496f2d-2703-4d5e-9c2d-bbae11d6d359]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:55.277830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-6f2b0b43-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:55.321033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-6f2b0b43-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:55.321773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-6f2b0b43-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:55.322232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-6f2b0b43-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:55.322684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-6f2b0b43-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:55.323201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-6f2b0b43-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6f2b0b43-44c3-4f4a-8c14-3638c1c80ebb', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:dc:e5:61'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:55.393607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19afd853-a274-4745-bde3-9bdab0c25a2b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:55.464008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a5259b21-09e5-47a3-8c69-b9f143498de2]: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:55.464008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:55.464008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:55.464008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:55.464008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:55.464008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:55.464008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:55.464008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:55.464008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:55.464008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:55.464008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:55.464008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:55.464008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:55.465969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5259b21-09e5-47a3-8c69-b9f143498de2]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:55.466530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:55.467016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:26:55.514668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-399bcdd7-ebc4-4ff7-9a4e-fa0dcd7621a6 tempest-RoutersAdminNegativeIpV6Test-2084173025 tempest-RoutersAdminNegativeIpV6Test-2084173025-project-admin] Got router deleted notification for 08ef7f99-cc29-48b7-81f0-d34c03161c15 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:26:55.515503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[516a3c4b-a55d-4d63-9bc3-6482795b3a1a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:55.518504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 08ef7f99-cc29-48b7-81f0-d34c03161c15, action 1, priority 1, update_id d73ab7e4-d5ce-44c4-b333-be47b9522d84. Wait time elapsed: 0.003 Jul 01 09:26:55.518976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 08ef7f99-cc29-48b7-81f0-d34c03161c15, action 1, priority 1, update_id d73ab7e4-d5ce-44c4-b333-be47b9522d84. Wait time elapsed: 0.004 Jul 01 09:26:55.521586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 08ef7f99-cc29-48b7-81f0-d34c03161c15 was not found. Performing router cleanup Jul 01 09:26:55.540197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[044b2741-2e87-46aa-a51d-632a15be5e71]: (4, ['qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:55.541492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:55.543907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:55.544446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 08ef7f99-cc29-48b7-81f0-d34c03161c15, update_id d73ab7e4-d5ce-44c4-b333-be47b9522d84. Time elapsed: 0.027 Jul 01 09:26:55.819397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d65481ae-086f-4101-a967-a8d4039b752a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:55.889908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80982aab-f3ea-43c6-8e4a-59167f7f5d11]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.132440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.622s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:56.133670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:56.137823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:56.138552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:56.139245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:56.142035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ac63b10-b52f-4178-ba55-6f91922394a6]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.150692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18fc7237-8adc-490a-a503-86b19d148c33]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.162224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:26:56.163398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:56.163910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:56.168389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" released by "process_external" :: held 0.971s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:56.171760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:56.178161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:56.184985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[452a5a13-feca-4148-b5d9-3771d18ad4d9]: (4, ['qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.185556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3d67c0d-9c36-4f8c-a60f-4eeefe43ec0a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.186273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:56.186629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:56.186887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:56.187231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:56.191063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cb8e255-cfc3-4438-af97-eb7728e64900]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.194647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e86747b0-e6e1-4e5d-85ae-0507ae1e0d97]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.197299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fc32cc8-b64f-49ab-af17-37b1f344a7b8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.205424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4ca3339-7b2d-4b61-b571-8085d3cadf9d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.235969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c50a76c6-46e7-44f4-b93e-cbb160b07ce9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126478, 'error': None, 'target': 'qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.242748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c47cbd26-afe5-4d95-b2cb-ac9ea09b925b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251120, 'reachable_time': 29384, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126477, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.250919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:56.251391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed686eb3-2052-47e2-ab68-91b5343f40ad]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:56.253050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-snat-router-969169340", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f6303cc2036a435290b0447468eda290", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": false Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f2b0b43-44c3-4f4a-8c14-3638c1c80ebb", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:37Z", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:40Z", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f6303cc2036a435290b0447468eda290", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f2b0b43-44c3-4f4a-8c14-3638c1c80ebb", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dc:e5:61", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 591, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:39Z", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:50Z", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": false, Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:26:56.257610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:56.271672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06, action 3, priority 2, update_id f6564dc8-dda1-44fc-b08d-9f3f49ee76af. Wait time elapsed: 12.732 Jul 01 09:26:56.272991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06, action 3, priority 2, update_id f6564dc8-dda1-44fc-b08d-9f3f49ee76af. Wait time elapsed: 12.733 Jul 01 09:26:56.273548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:26:56.306360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[830e4b56-0ca8-4a03-b1ef-162ac148a3eb]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.341117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff32180a-7c41-4cd8-8652-e6163be6cf53]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.383194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2191c31c-e3eb-48fc-862a-06d2c51e54b2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.451394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb437294-f0d3-4823-bb30-02d41a2482d0]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.451963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:56.452369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:56.452646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:56.488840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24028bf3-ec2a-4085-ad61-f838d918741c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251250, 'reachable_time': 30684, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126489, 'error': None, 'target': 'qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.500658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:56.502481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:56.503024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:56.503596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:26:56.574579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9febdcba-ddea-4e2d-ac42-e3d191794022]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251250, 'reachable_time': 30684, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126501, 'error': None, 'target': 'qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.575631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:56.576367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:26:56.612887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb6f958f-02f5-44ba-af4d-0282fb4cea5c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.850170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72e9ae7d-78e3-4570-8c1e-0e0386cb238b]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:56.887036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cbce533-4822-45e5-9c81-9f49833d8b00]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.061563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0980e1c2-e17d-4dbc-9151-864330e491dc]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.062314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:26:57.062791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:57.063227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:57.063720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" released by "process_external" :: held 0.560s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:26:57.064365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:57.064951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" acquired by "process_address_scope" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:26:57.065542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:57.066405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:26:57.103472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73399c43-3299-4f7d-b592-4d010bed4ef5]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.173861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32cf6cf4-7b82-4b66-be00-5ca06f4c85ca]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.223517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dca2a718-8893-41be-9eef-d07d9678c5d7]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.281749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd1c86be-e66a-40bf-a576-97ac1f0731b4]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.282242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:26:57.282606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:57.282871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:57.283157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" released by "process_address_scope" :: held 0.218s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:26:57.284041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:57.284694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:57.285115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:26:57.285629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:57.286738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:26:57.340111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3633275-f2a0-429c-a213-d8acfee30671]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.414248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1dce86bf-a5a5-4738-9bbe-f35afc9c7e4a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.486760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad213257-503a-4855-a8d5-4e6546fefbc5]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:26:57 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:26:57 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.567695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0d6f3c0-e407-48c9-9db9-f17aad586c78]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.569600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:26:57.570067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:57.570476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:57.570902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74" released by "apply_metadata_nat_rules" :: held 0.286s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:26:57.571565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.pid.haproxy Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 Jul 01 09:26:57.572542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:26:57.575215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:26:57.646572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-517e49d3-eb0e-46cc-9eaf-0c848cf1183e tempest-RoutersAdminTest-1421187323 tempest-RoutersAdminTest-1421187323-project-admin] Got router deleted notification for ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:26:57.689090 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74[126550]: Proxy listener started. Jul 01 09:26:57.709469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:26:57.709936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74, update_id 40c1cac3-2f90-4bad-88c5-a64787eaf4c6. Time elapsed: 3.200 Jul 01 09:26:57.710324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74, action 1, priority 1, update_id 19791dc0-17ee-4d4f-b817-a9cb7f0be226. Wait time elapsed: 0.063 Jul 01 09:26:57.710663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74, action 1, priority 1, update_id 19791dc0-17ee-4d4f-b817-a9cb7f0be226. Wait time elapsed: 0.064 Jul 01 09:26:57.711203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:26:57.711767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:57.749451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[869a6858-cbd5-4842-8c13-e1b7cb80d130]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.751050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[083d86f0-9e59-46c2-b670-85a83f0c66e2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.752068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ecf65ac-18ae-4a12-b829-37a54059e0b9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.752535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:26:57.753553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b283c0c-6ed1-40e4-83b9-a76e6a7e0395]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.795026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d4050be-7f3b-4dce-87e7-8a51355d1cf2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251250, 'reachable_time': 30684, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126565, 'error': None, 'target': 'qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.796127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:57.796629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:57.844794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73f2ee70-4791-4168-a5c1-3b4d14cfff41]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251250, 'reachable_time': 30684, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126567, 'error': None, 'target': 'qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.847184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:26:57.847872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:57.848337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:57.849424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:57.850945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:26:57.852279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[453fd2b5-2d0f-444f-8b40-671e7177d80d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.894935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.621s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:26:57.896092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:26:57.897386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:57.897940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:57.898458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:57.900783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0cf06e54-6f58-4bd1-8e67-c439cf39787b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.916638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1df38e7-4b11-4d22-a370-33adf99825f1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251250, 'reachable_time': 30684, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126570, 'error': None, 'target': 'qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.927065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:57.927747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e098464a-13f3-4f26-9241-34e5a97353f9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:57.928417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:26:57.929090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:26:57.929648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:57.930146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:57.930777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:57.931429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:57.932102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for ef2471b5-92c7-49d9-a8ef-0d1fb0c0ae74, update_id 19791dc0-17ee-4d4f-b817-a9cb7f0be226. Time elapsed: 0.222 Jul 01 09:26:57.966004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78878c8e-678b-4e00-be54-5475bbdceb4c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.002551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50a0a21f-4188-486b-93e6-50bded54a5df]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.060774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0be4a899-d900-4713-a4cd-cbc6e13a8951]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126574, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.098615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be0e7bf6-8a37-4cb7-93fc-db4c608394a6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.132993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bffca5c4-187f-4bec-a010-987fc96e1ea9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.175233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[808b25af-ed02-4239-951b-dc928eee76de]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.215134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8f4c843-d895-43d7-81cf-98eb3be19397]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.215768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:58.216231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:58.217450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:26:58.262764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5095f9aa-b194-404d-8852-45716096f46e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251432, 'reachable_time': 37396, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126580, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.263818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:58.264260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:58.265925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:58.265925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:26:58.265925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:26:58.265925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:26:58.265925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '6f2b0b43-44c3-4f4a-8c14-3638c1c80ebb', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:dc:e5:61', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'device_owner': 'network:router_gateway', 'standard_attr_id': 591, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.9', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:26:39Z', 'updated_at': '2022-07-01T09:26:50Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-6f2b0b43-44), ns(qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:26:58.307404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:26:58.307825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a812f97-0f82-4e4a-baeb-60444bbdf791]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.311084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a05f0560-3101-43f0-8530-f2d3dfd6f189]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.312206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-6f2b0b43-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:58.356359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-6f2b0b43-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:58.357734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-6f2b0b43-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:58.358277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-6f2b0b43-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:58.358810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-6f2b0b43-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:58.359337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-6f2b0b43-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6f2b0b43-44c3-4f4a-8c14-3638c1c80ebb', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:dc:e5:61'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:26:58.406931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dba05e6e-25c4-44f4-b4ae-5f9a389ea752]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.492839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9f7c9b3b-f419-4a9f-a47c-5dcf0fc0cce7]: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:26:58.492839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:26:58.492839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:26:58.492839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:26:58.492839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:26:58.492839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:26:58.492839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:26:58.492839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:26:58.492839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:26:58.492839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:26:58.492839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:26:58.492839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:26:58.492839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:58.494266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f7c9b3b-f419-4a9f-a47c-5dcf0fc0cce7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.495196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:26:58.495835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:26:58.530199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af634b76-be77-4da6-a3e7-f35f51edbb5d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.779031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d830d9ee-ea32-4007-a049-3821af365063]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:58.815500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36cf640d-eb27-4303-bc3b-b4921a1054fa]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:59.021958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7d45de1-0fa6-4652-8753-21cc9aca51cc]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:59.022941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:26:59.023273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:59.023900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:26:59.024336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" released by "process_external" :: held 0.759s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:59.025414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:59.028873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:59.035301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b154f90-a274-46d9-b69c-f08b5a416d5a]: (4, ['qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:59.037450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:59.038407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:59.038813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:26:59.039653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:26:59.041781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea98cf08-cb1e-4497-92e7-8eda3b79c2cf]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:59.043865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[720f98a1-ce22-4507-9997-dd948c0cb92a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:59.045639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fba895f-777c-439e-b0f8-b93bd7f5cf75]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:59.085306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00f45cd9-0746-4f76-8e82-cdfa76dbcbfd]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251432, 'reachable_time': 37396, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126598, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:59.096899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:26:59.097314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8e9e9cc-3ba5-4194-a1df-fd613f8ec226]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f2b0b43-44 not found in namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06. Jul 01 09:26:59.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-snat-router-969169340", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f6303cc2036a435290b0447468eda290", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": false Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f2b0b43-44c3-4f4a-8c14-3638c1c80ebb", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:37Z", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:40Z", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f6303cc2036a435290b0447468eda290", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f2b0b43-44c3-4f4a-8c14-3638c1c80ebb", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dc:e5:61", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 591, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:39Z", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:50Z", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": false, Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:26:59.104579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:26:59.116370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06, action 3, priority 2, update_id f6564dc8-dda1-44fc-b08d-9f3f49ee76af. Wait time elapsed: 15.576 Jul 01 09:26:59.116808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06, action 3, priority 2, update_id f6564dc8-dda1-44fc-b08d-9f3f49ee76af. Wait time elapsed: 15.577 Jul 01 09:26:59.117277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:00.824146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.706s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:00.825016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:00.826189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:00.826327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:00.827254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:00.830535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[795158a4-749d-459d-b6c7-a0569f0e6dac]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:00.856407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb679b1d-c84a-4a31-953f-3c38e6b86cdf]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:00.893356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0e0e54c-0533-4c8c-8fd0-72cb48dde5b7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:00.952638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02580ab1-c287-44d0-82e2-93571769e2a7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126605, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:00.996380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e091831d-b67b-4970-b396-827b01522601]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.055113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4357f6d4-055e-461a-ae8e-006537bb85c5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.115309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[910915e7-dbfc-457f-b42c-d2a33f2f2137]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.154546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26d348f7-c0a1-4bdb-9985-c822c523011c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.155404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:01.156354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:01.156769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:01.190709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f53b944-ec2e-48ec-8e15-69717f901a88]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251721, 'reachable_time': 25288, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126643, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.192124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:01.193110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:01.193938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:01.194369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:27:01.232150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5f3b604-b9c7-4354-9df0-cb192fac5e89]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251721, 'reachable_time': 25288, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126644, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.233227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:01.233844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:27:01.272531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8b0e7ec-9ff6-4e53-b5f0-60ad5b9f3d11]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.465075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f6e8d2c-326b-49e1-ba20-810531816799]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.496519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc416dda-9417-4284-8fd9-96f0fb7e1d4d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.733801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20abbfc5-4011-4eef-bd26-0bb48c388582]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.734347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:27:01.734613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:01.734894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:01.735184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" released by "process_external" :: held 0.541s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:27:01.737445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:01.738383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" acquired by "process_address_scope" :: waited 0.002s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:27:01.738844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:01.739406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:27:01.799375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6e81d10-20fd-4932-a849-fc0e50ced9ce]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:01 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:01 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:01 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:27:01 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:01 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:27:01 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:01 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:27:01 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.870369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3d06b38-99de-49d2-aa79-a580800a5afb]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.907188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[165fedca-43cc-4d90-8340-7128e190940c]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:01 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:01 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:01 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:27:01 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:01 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:27:01 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:01 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:27:01 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.949853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d382a335-30e1-4355-9c3b-9178e05135a1]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:01.950358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:27:01.950697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:01.950939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:01.951227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" released by "process_address_scope" :: held 0.213s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:27:01.951640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:01.952806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:01.953226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" acquired by "apply_metadata_nat_rules" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:27:01.953865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:01.954461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:27:02.017641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[454567e8-1d37-4e4c-bc67-bdb4081584c8]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:01 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:01 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:01 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:27:01 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:01 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:27:01 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:01 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:27:01 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:02.110733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[048dad95-76d7-4dd1-b795-7b97cc58a70d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:02.162867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e769d841-289c-42ca-ad16-a0de3a3b01c4]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:02 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:02 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:02 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:27:02 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:02 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:27:02 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:02 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:27:02 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:02.234905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8142b5dc-af35-4d7d-8d27-a896ff0899ef]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:02.237825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:27:02.238135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:02.238400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:02.238688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06" released by "apply_metadata_nat_rules" :: held 0.285s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:27:02.239514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.haproxy Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 Jul 01 09:27:02.240388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:27:02.242384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:27:02.324828 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06[126676]: Proxy listener started. Jul 01 09:27:02.338437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:27:02.338956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06, update_id f6564dc8-dda1-44fc-b08d-9f3f49ee76af. Time elapsed: 3.224 Jul 01 09:27:03.229000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-4569f585-487a-4812-907f-2d8009b4651f tempest-RoutersAdminTest-1421187323 tempest-RoutersAdminTest-1421187323-project-admin] Got router deleted notification for 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:27:03.229614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06, action 1, priority 1, update_id 2b02dbd0-4d9f-4626-8a2f-3717ab54744c. Wait time elapsed: 0.000 Jul 01 09:27:03.229816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06, action 1, priority 1, update_id 2b02dbd0-4d9f-4626-8a2f-3717ab54744c. Wait time elapsed: 0.001 Jul 01 09:27:03.230128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:27:03.235166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:03.286596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc2e25b4-3cf7-4f2d-8b9e-d1527ff60305]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:03.289306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[365c7569-8ebf-4822-9c68-27d36509b6a8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:03.290646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[931db99a-65d1-4ca6-86cf-997540393cee]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:03.291273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:27:03.292306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b93be0c-ac42-4512-b024-176b1ff93a32]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:03.337691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8fa34ae-f554-4cd1-b71e-811554c0bb5e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251721, 'reachable_time': 25288, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126685, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:03.341944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:03.342220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:03.379698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[549562e0-d760-4336-bad8-8ac9d8ed9b40]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251721, 'reachable_time': 25288, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126686, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:03.380591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:27:03.381139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:03.381561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:03.381815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:03.382648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:27:03.383847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b4f60db-6bcf-4f72-9860-c83369c68b68]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:03.419244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74b5a08a-db9e-45fd-a9a6-c734639c6d0e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 251721, 'reachable_time': 25288, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126687, 'error': None, 'target': 'qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:03.430234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:03.430546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eee02e94-b31f-4ac0-b4b1-37409c0288aa]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:03.431197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:27:03.432167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:03.432512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:03.432786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:03.433082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:03.433342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:03.433587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 7aa56a2a-0aa2-4b96-b0c3-a40ff1977c06, update_id 2b02dbd0-4d9f-4626-8a2f-3717ab54744c. Time elapsed: 0.204 Jul 01 09:27:04.753625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-554e50b5-6574-422c-add3-47c9161342c5 tempest-ServersTestJSON-1669246664 tempest-ServersTestJSON-1669246664-project] Got routers updated notification :['4da8db80-0643-4392-ad3e-03f5c3888347'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:04.754525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 1, update_id f1931dc6-0f2a-4abc-9a94-001f8baec952. Wait time elapsed: 0.001 Jul 01 09:27:04.754984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 1, update_id f1931dc6-0f2a-4abc-9a94-001f8baec952. Wait time elapsed: 0.001 Jul 01 09:27:04.755267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:05.004522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-4dc66fa0-e09a-4a38-aa89-6cdf03730d47 tempest-ServerActionsTestJSON-77649674 tempest-ServerActionsTestJSON-77649674-project] Got routers updated notification :['9974ba83-7306-4b67-8057-d6ab904a1c00'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:05.006230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 1, update_id 21799d67-90be-4540-8039-89081fc0472b. Wait time elapsed: 0.001 Jul 01 09:27:05.006626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 1, update_id 21799d67-90be-4540-8039-89081fc0472b. Wait time elapsed: 0.001 Jul 01 09:27:05.007143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:06.701315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.946s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:06.701951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:06.702545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:06.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:06.703158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:06.712461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b459975-8ab4-4eab-a4fa-13ee9804e1ff]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:06.743425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c687d29a-11da-4484-8d32-eece5c6af6bd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:06.802083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e318a9b-85cc-4b20-843f-5169e6d75794]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:06.861078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7aa80c40-ab16-4ffa-8e42-a783775a5902]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126693, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:06.904549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5548b3c9-0ada-4b1c-8685-13977c900ea1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:06.936588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bdc81cd2-09f9-4205-a297-4299bec101d6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:06.976773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d37b734-ee45-44b7-b04a-d67e535709b2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:07.012752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81bcab57-cb50-4588-ace2-d64ce4686b64]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:07.013510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:07.013948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:07.014361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:07.014977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:07.051787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:07.052903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed89069f-f222-4ab4-a98f-6111dc7e5fd8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:07.055968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af1c8487-60c9-47ad-862f-89bc39f2251c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:07.057132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:07.090886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:07.090886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:07.090886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:07.090886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:07.091636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:07.157991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca8137d9-ea29-470a-9b00-2392a49e00c4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:07.213138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[383b8f43-0bd4-4105-90ce-2c1e98b3cdbb]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:07.213138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:07.213138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:07.213138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:07.213138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:07.213138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:07.213138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:07.213138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:07.213138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:07.213138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:07.213138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:07.213138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:07.213138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:07.215814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[383b8f43-0bd4-4105-90ce-2c1e98b3cdbb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:07.217074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:07.225058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:07.228996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8070fc41-52bb-4aa3-bbc4-54aa0dca12c8]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:07.233377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:07.233824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:07.234087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:07.235173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:07.237954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[702eaa9a-5b9f-4149-b8a6-32a10765e8d0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:07.239770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f75251c-e86b-4d0c-94db-16af4da2f087]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:07.243442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8fe47645-5a60-4f05-9dc4-9de751e2f140]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:07.279840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ecac5d7-7d7c-4ed1-8c8a-c455701271e1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 252312, 'reachable_time': 15940, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126714, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:07.287554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:07.287951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7dd9c869-4117-447e-b372-5edd5494a306]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:07.289299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8e7d7bd8-acc2-4356-b50e-ff776e5c8d67", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.40", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "20d9e825-4027-4a35-987e-9fbcec875296", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.12", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 522, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:43:53:56", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4736091a-3a94-4af2-9e3d-4d661a5d3f11", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:12Z", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:01Z", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:07.292346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:02Z", Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 13, Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id f1931dc6-0f2a-4abc-9a94-001f8baec952. Wait time elapsed: 2.538 Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id f1931dc6-0f2a-4abc-9a94-001f8baec952. Wait time elapsed: 2.538 Jul 01 09:27:07.314857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:08.649715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.356s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:08.650343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:08.650943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:08.651290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:08.651627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:08.653458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97793046-ee2a-4632-953c-003ee4db0ba9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:08.675536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c507e870-0cab-40be-bd92-e50ef02c9837]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:08.716564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d30946a-2978-4543-83f1-f0f4ebc9e81e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:08.776528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38f2c4a3-86ea-4968-a298-df154d3e5abf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126717, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:08.825395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2f41f3a-47a5-45f5-94fa-9a32357ef51f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:08.868558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4b3957d-e0a8-43eb-a7bd-ac543c8c9b7b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:08.914633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98b30c6a-6428-46d4-819b-e355f37ad24f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:09.006786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1515864-dac0-42c1-828c-1e73ec61e89c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:09.007922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:09.008341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:09.008757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:09.009226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:09.076692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:09.076881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f71fca3-3cdc-4c09-966e-51ad8c51642a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:09.081465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab9c8eac-777a-4412-85ac-065b5dd2acdf]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:09.084121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:09.131250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:09.132128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:09.133596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:09.137425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:09.142010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:09.207339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0d34488-ebb2-4840-bfed-fbb332a4d56d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:09.297872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ec2d8c42-0f0a-4596-8568-5f6d811974e5]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:09.297872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:09.297872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:09.297872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:09.297872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:09.297872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:09.297872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:09.297872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:09.297872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:09.297872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:09.297872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:09.297872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:09.297872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:09.299277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec2d8c42-0f0a-4596-8568-5f6d811974e5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:09.300525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:09.304341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:09.310304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcdc8d7c-a07b-4049-928d-0efc3c3eb101]: (4, ['qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:09.311158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:09.313145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:09.313880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:09.314441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:09.322570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa55eaf8-a4fc-4063-83c8-3bb75148c40e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:09.324186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7889471-8b72-46ab-8265-17fabdd9645d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:09.327321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ef7e932-c6e4-48f8-a5dc-1979f0a8082e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:09.372593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93de31f0-581b-46b4-a74b-62f2a2b0fa74]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 252504, 'reachable_time': 44700, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126769, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:09.383285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:09.383613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c5567a6-1fb9-4f40-b3cc-0b3e5f349150]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:09.385205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8e7d7bd8-acc2-4356-b50e-ff776e5c8d67", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.40", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "20d9e825-4027-4a35-987e-9fbcec875296", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.12", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 522, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:43:53:56", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4736091a-3a94-4af2-9e3d-4d661a5d3f11", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:12Z", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:01Z", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:09.387963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:02Z", Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 13, Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id f1931dc6-0f2a-4abc-9a94-001f8baec952. Wait time elapsed: 4.632 Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id f1931dc6-0f2a-4abc-9a94-001f8baec952. Wait time elapsed: 4.633 Jul 01 09:27:09.413081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:10.553122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.544s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:10.555499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:10.557461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:10.558818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:10.560112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:10.565418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e15f1441-d265-4a22-82b8-9b3cd4f3141f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:10.587144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[917da4a9-0538-47cf-958b-7ce7c65ba7e1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:10.633820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06440b32-ab38-4a11-8f34-d3280c382a54]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:10.698817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34aa86cf-bd11-4b99-8fcd-5f5b565d1f6e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126804, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:10.727947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ed35c2b-4cb4-4f74-95c6-97c31ec14fd2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:10.763115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf5baa8e-bf4a-49ae-bb7b-13d36ab53628]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:10.784258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6bf3e80-da9c-48c9-8ecf-8e2c1e0cafc6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:10.808141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1657594d-2317-476f-9e77-401d17e78e7d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:10.809736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:10.810297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:10.810880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:10.812150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:10.847737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:10.849643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62e6bda8-8040-4747-8ddc-c925e4a5bf17]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:10.853891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8326a8d7-65a3-44eb-a1e5-4b2169de1444]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:10.855715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:10.886264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:10.887152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:10.887793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:10.889543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:10.890160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:10.902151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-2fa5f707-cb11-4e7d-9ab8-2f146eccfae8 tempest-RoutersAdminTest-1423367726 tempest-RoutersAdminTest-1423367726-project] Got routers updated notification :['fe306376-5e62-4867-88bd-cfdc3b1f8bac'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:10.948205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fe306376-5e62-4867-88bd-cfdc3b1f8bac, action 3, priority 1, update_id 511d519a-7282-4b69-b5f7-41df7f88c6c2. Wait time elapsed: 0.046 Jul 01 09:27:10.948460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fe306376-5e62-4867-88bd-cfdc3b1f8bac, action 3, priority 1, update_id 511d519a-7282-4b69-b5f7-41df7f88c6c2. Wait time elapsed: 0.046 Jul 01 09:27:10.948772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:10.983774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26a5bd5c-8e09-4a8e-a107-af108310404d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:11.054855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[40a3ad9c-e696-4434-830e-b77acf4a0bcb]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:11.054855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:11.054855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:11.054855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:11.054855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:11.054855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:11.054855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:11.054855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:11.054855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:11.054855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:11.054855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:11.054855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:11.054855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:11.055957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40a3ad9c-e696-4434-830e-b77acf4a0bcb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:11.056203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:11.058081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:11.061064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5e71bf1-5a88-461c-a34e-1c7c45041b50]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:11.061064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:11.061064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:11.061064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:11.061064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:11.062581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05d22a08-d890-4579-80ae-c2dd9eadeaa1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:11.064242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6666e31-54a8-4b01-9c26-e45b716a00e5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:11.066520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff41651f-5f93-485e-bf9d-67224c24607e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:11.111790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49261931-bf5c-4913-b256-d9bfe2828b43]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 252696, 'reachable_time': 28056, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126823, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:11.117575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:11.117864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[344d983b-62ce-4044-98d0-7a29468a5166]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:11.119163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ff4cb520-36eb-4815-860e-ab86ac8aecec", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.192", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "6612ab5f-3352-4254-8460-dac56322c900", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 525, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:d8:4f", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "672c983b-c364-46e8-90d8-0553b70b3325", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:13Z", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:02Z", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:11.122437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:01Z", Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 21799d67-90be-4540-8039-89081fc0472b. Wait time elapsed: 6.115 Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 21799d67-90be-4540-8039-89081fc0472b. Wait time elapsed: 6.115 Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.184s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router fe306376-5e62-4867-88bd-cfdc3b1f8bac was not found. Performing router cleanup Jul 01 09:27:11.136824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5961939-9a59-4ea7-84c7-6e2dbfdf9f04]: (4, ['qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:11.139699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:11.139699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:11.139699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for fe306376-5e62-4867-88bd-cfdc3b1f8bac, update_id 511d519a-7282-4b69-b5f7-41df7f88c6c2. Time elapsed: 0.190 Jul 01 09:27:13.346034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-d55c4b35-6458-48cf-9794-93e8a72aeaaf tempest-RoutersAdminTest-1421187323 tempest-RoutersAdminTest-1421187323-project-admin] Got routers updated notification :['fe306376-5e62-4867-88bd-cfdc3b1f8bac'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:13.348029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fe306376-5e62-4867-88bd-cfdc3b1f8bac, action 3, priority 1, update_id 8f1d0111-efde-413e-87fc-b3d576ac99b8. Wait time elapsed: 0.001 Jul 01 09:27:13.349039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fe306376-5e62-4867-88bd-cfdc3b1f8bac, action 3, priority 1, update_id 8f1d0111-efde-413e-87fc-b3d576ac99b8. Wait time elapsed: 0.001 Jul 01 09:27:13.349474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:13.617435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.268s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:13.617786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router fe306376-5e62-4867-88bd-cfdc3b1f8bac was not found. Performing router cleanup Jul 01 09:27:13.618897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e113d9b-fe26-48f8-9486-4f8f0f4e590a]: (4, ['qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:13.619578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:13.620030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:13.620432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for fe306376-5e62-4867-88bd-cfdc3b1f8bac, update_id 8f1d0111-efde-413e-87fc-b3d576ac99b8. Time elapsed: 0.272 Jul 01 09:27:15.015819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.628s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:15.018041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:15.019895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:15.025447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:15.031455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.006s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:15.040651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a02c3aa4-fd8f-4450-8c68-e481c9d8b55f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.073834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:15.075469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:15.076695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:15.078260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:15.109146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a17139fc-445e-4ca8-96ba-b163e5b615aa]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.144934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d089a4e-364c-491a-9a60-a5ea3cc95ac2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.151743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.030s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:15.151954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:15.153036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:15.153564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:15.153899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:15.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0fed29e-6344-40ba-9b65-4feddc5f85b5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.179667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a17f687f-9706-4d88-9ade-6c461fff683d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126830, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.183822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a22740d8-eb5f-43ae-8fed-7181b8614334]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.216120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c740037-9b2e-4582-83fd-a6dfab91acb5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.216411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95ca941c-3927-42db-b0c6-2ed18b622bbf]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.258688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99996f50-6880-4457-b38f-97dad52aba9f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.274510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fa49ce4-26f0-41ea-8a2a-e4774c6b6306]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126835, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.334919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72684d45-0faa-4f68-98da-261a4257cf84]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.336015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a27d9898-996c-4d54-a7ba-ea0e43e57cb6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.398656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee7efa08-89b9-4f39-8c68-48b85ee8789b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.402665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:15.403185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:15.403766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:15.404266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:15.439309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:15.439830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45c6f3ab-7509-48ec-a598-75975ebd5f04]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.446106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cefa13a9-482b-45c7-bce5-c78c94182f28]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.450625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf3b034c-e957-48fd-9628-f569d29e5513]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.462713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:15.497861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:15.498528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:15.498907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:15.499295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:15.499704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:15.537210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[267a0317-02c3-4c76-8916-93488ca98b83]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.538664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82d3459d-8a7d-4e8d-8b3c-c9dc098c72bf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.613408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[37b83235-42f4-4873-a018-32f5589441e0]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:15.613408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:15.613408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:15.613408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:15.613408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:15.613408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:15.613408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:15.613408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:15.613408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:15.613408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:15.613408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:15.613408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:15.613408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:15.614970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37b83235-42f4-4873-a018-32f5589441e0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:15.615304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:15.617319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:15.620870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a012466-8050-45c3-94e9-2004e742e63c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.623689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:15.624253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.005s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:15.624674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:15.625860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:15.634673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f161c606-e69c-4e4d-ab00-4e308650622f]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.637933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:15.638256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:15.638514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:15.638852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:15.642630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf2d0595-472d-4e31-9efd-73faa71f6a6a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.644668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e4592dd-7825-4a8a-bf6f-4c1ca769d15b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.648072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8086ff08-3fc9-4560-b4d0-00d0a6916aa0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.664815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:15.664815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2864e25-aba8-4081-b4e7-3ebd46b5b491]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.684526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cecd002f-ecc0-4c9a-a600-5acc7b267822]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.685551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:15.697362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[279092dc-c1a7-4cea-a568-f3b76904955f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 253144, 'reachable_time': 37924, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126876, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.701148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:15.701593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0402539-c36c-4b75-97dd-7ec22ef3bf7c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:15.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8e7d7bd8-acc2-4356-b50e-ff776e5c8d67", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.40", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "20d9e825-4027-4a35-987e-9fbcec875296", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.12", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 522, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:43:53:56", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4736091a-3a94-4af2-9e3d-4d661a5d3f11", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:12Z", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:01Z", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:15.708156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:10Z", Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:15.719228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:15.729595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id f1931dc6-0f2a-4abc-9a94-001f8baec952. Wait time elapsed: 10.968 Jul 01 09:27:15.729595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id f1931dc6-0f2a-4abc-9a94-001f8baec952. Wait time elapsed: 10.968 Jul 01 09:27:15.729595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:15.729595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:15.729595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:15.729595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:15.729595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:15.729595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:15.794006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab1d3015-322f-4023-9a92-a3029803415f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.838882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[26bf4be9-4cb1-471a-ad3a-43838d13ada6]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:15.838882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:15.838882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:15.838882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:15.838882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:15.838882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:15.838882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:15.838882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:15.838882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:15.838882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:15.838882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:15.838882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:15.838882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26bf4be9-4cb1-471a-ad3a-43838d13ada6]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:15.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:15.841890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6cc8c50a-b048-46f7-a8b5-b15f29bc43e0]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.847509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:15.847509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:15.847509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:15.847509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:15.847509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7c4b163-2c80-4296-9548-0fd4d1666066]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.847509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[701f12fc-bffa-471b-99df-a5631da16420]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.849440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14857ace-c04b-424f-8bf5-7bc75bde98ba]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.876987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41fe119a-4332-442a-92ce-68c3abcf6cc2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 253153, 'reachable_time': 31324, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126888, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.884814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:15.885076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a962eaf-9734-4c16-b76b-224aeab33f8c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:15.886118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ff4cb520-36eb-4815-860e-ab86ac8aecec", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.192", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "6612ab5f-3352-4254-8460-dac56322c900", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 525, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:d8:4f", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "672c983b-c364-46e8-90d8-0553b70b3325", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:13Z", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:02Z", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:15.888930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:01Z", Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 21799d67-90be-4540-8039-89081fc0472b. Wait time elapsed: 10.882 Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 21799d67-90be-4540-8039-89081fc0472b. Wait time elapsed: 10.882 Jul 01 09:27:15.912482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:16.056224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-fc5a4bf0-21e3-40db-bd35-50614d829d67 tempest-RoutersAdminTest-1423367726 tempest-RoutersAdminTest-1423367726-project] Got router deleted notification for fe306376-5e62-4867-88bd-cfdc3b1f8bac {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:27:16.068194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fe306376-5e62-4867-88bd-cfdc3b1f8bac, action 1, priority 1, update_id 6d02e35b-d914-4715-8f99-a2d19d4df840. Wait time elapsed: 0.008 Jul 01 09:27:16.069043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fe306376-5e62-4867-88bd-cfdc3b1f8bac, action 1, priority 1, update_id 6d02e35b-d914-4715-8f99-a2d19d4df840. Wait time elapsed: 0.009 Jul 01 09:27:16.069910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router fe306376-5e62-4867-88bd-cfdc3b1f8bac was not found. Performing router cleanup Jul 01 09:27:16.071080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63d84799-1a43-48b3-8ecf-ab7403b720b6]: (4, ['qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:16.071745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:16.072339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:16.072791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for fe306376-5e62-4867-88bd-cfdc3b1f8bac, update_id 6d02e35b-d914-4715-8f99-a2d19d4df840. Time elapsed: 0.005 Jul 01 09:27:16.963927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.076s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:16.964502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:16.965606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:16.966086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:16.966530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:16.967946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a847d86d-1195-4e3c-9ce1-9d2108e22110]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:16.988352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c565f0cc-e223-45e5-9e26-829a4d233ce9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.012262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9892a1f2-cc21-4520-817b-56d8ad2bb90b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.065937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09262046-4080-4b74-89e6-34653b13d5ac]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126891, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.102404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9551412d-6464-4f34-a8c4-0ec44c480d75]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.143200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f010146-fcf9-4bdd-9c5a-126c59662a1d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.173851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a3f16c5-df0b-4c4a-aae4-d5a2daec19c0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.197655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0c505f0-f035-444c-8716-aa8c538920a7]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.205185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:17.205767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:17.206368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:17.206849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:17.231947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:17.232812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da97f71c-e8ff-4169-96eb-10c0c62fc89b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.237167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ffabc47-6750-49d6-b903-02a46567c622]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.238362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:17.268317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:17.269393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:17.270061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:17.271483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:17.272111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:17.303116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4a2ff04-6c95-479a-913f-c5f3cdf52e23]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.365284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f6b8d6ea-ae17-4108-a86f-ecc5189f8113]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:17.365284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:17.365284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:17.365284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:17.365284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:17.365284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:17.365284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:17.365284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:17.365284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:17.365284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:17.365284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:17.365284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:17.365284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:17.366305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6b8d6ea-ae17-4108-a86f-ecc5189f8113]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:17.367016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:17.369962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:17.372221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29362296-c21a-4c51-abc3-b804f5535963]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.374110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:17.375651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:17.376098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:17.376586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:17.378065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a714ba46-9543-4159-a007-55cabb702db2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.380096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aaf3bf26-156b-4098-971b-b771aebbba36]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.383187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05de9142-82a7-4c07-bf15-641c00bc40e3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.409933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[708f6886-f674-42d1-b1e7-1d754f8b79e8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 253332, 'reachable_time': 27300, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126910, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.421907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:17.422238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0db08a7e-1919-43e5-beb9-729c9963c2e7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:17.424324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ff4cb520-36eb-4815-860e-ab86ac8aecec", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.192", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "6612ab5f-3352-4254-8460-dac56322c900", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 525, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:d8:4f", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "672c983b-c364-46e8-90d8-0553b70b3325", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:13Z", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:02Z", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:17.427928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:14Z", Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 21799d67-90be-4540-8039-89081fc0472b. Wait time elapsed: 12.421 Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 21799d67-90be-4540-8039-89081fc0472b. Wait time elapsed: 12.422 Jul 01 09:27:17.446680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:18.370489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-984c04b6-4777-496e-8e3b-0c038ddb18f2 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:18.372203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, action 3, priority 1, update_id 75c097b0-b161-4cdd-8125-1779a48fa6d4. Wait time elapsed: 0.001 Jul 01 09:27:18.372561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, action 3, priority 1, update_id 75c097b0-b161-4cdd-8125-1779a48fa6d4. Wait time elapsed: 0.002 Jul 01 09:27:18.372916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:18.562252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.189s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:18.562488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97 was not found. Performing router cleanup Jul 01 09:27:18.563992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efef6746-4f3e-458d-9e3c-bf7002648bb5]: (4, ['qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:18.564704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:18.565306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:18.565704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, update_id 75c097b0-b161-4cdd-8125-1779a48fa6d4. Time elapsed: 0.194 Jul 01 09:27:18.847848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.125s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:18.852683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:18.855199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:18.855948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:18.856468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:18.861322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c22c6096-1bf6-4a14-b5b3-4a9d458cfbeb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:18.899590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d02e186-c5a6-4576-852e-e9f2876439e6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:18.929700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a372fc88-b5b7-4efc-8290-374e0287468d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:18.985364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95c0d0eb-ecca-4561-8a7a-13d05467aeca]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126913, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.019676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37f244f4-0de8-46b1-a17d-ae414522f849]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.046306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a28d8c5-610c-4bda-b9d7-8d4459ce36b8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.089235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f379893f-f68d-48da-a0de-eb637f2f3e10]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.118836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13322564-a7fa-4dd7-bba2-eb42633a1a06]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.120752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:19.121265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:19.121747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:19.122744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:19.143811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:19.144328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[335633f9-a9e3-4c9e-8c59-6b6b4962f8a0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.148326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0af49bb-afa3-4118-8ab0-7988f2b98a5a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.149821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:19.184167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:19.185814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:19.186473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:19.187085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:19.187753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:19.231195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de14d6c2-35f2-4945-9a38-e750fff3313f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.272904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d497fe77-3f32-4fda-b3f3-5052ba40497e]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:19.272904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:19.272904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:19.272904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:19.272904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:19.272904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:19.272904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:19.272904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:19.272904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:19.272904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:19.272904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:19.272904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:19.272904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:19.273984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d497fe77-3f32-4fda-b3f3-5052ba40497e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:19.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:19.276936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:19.279263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[623b1802-0149-436f-bf97-9a892cb34d14]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.279263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:19.279263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:19.279718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:19.280233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:19.281444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13a0d93d-5631-4267-b8aa-1ed404ba1a9e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.282917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa85080b-c797-47cd-a73a-d427daea26bf]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.283867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[393ac9c7-212d-4dc2-899f-02165144ebeb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.312552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99080691-eda9-432e-aeb6-d02ddf1f3a00]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 253524, 'reachable_time': 39432, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126932, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.314934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:19.315380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43729d29-3494-4c9a-9df8-be4324d9b7ac]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:19.317944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8e7d7bd8-acc2-4356-b50e-ff776e5c8d67", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.40", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "20d9e825-4027-4a35-987e-9fbcec875296", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.12", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 522, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:43:53:56", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4736091a-3a94-4af2-9e3d-4d661a5d3f11", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:12Z", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:01Z", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:19.322366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:15Z", Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id f1931dc6-0f2a-4abc-9a94-001f8baec952. Wait time elapsed: 14.580 Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id f1931dc6-0f2a-4abc-9a94-001f8baec952. Wait time elapsed: 14.580 Jul 01 09:27:19.337827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:22.154559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.727s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:22.155619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:22.156940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:22.158153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:22.158702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:22.162382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce96fb78-72b2-47e4-9412-4fca21c83ac7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.187087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56b23cde-5c44-450c-86e0-ab87c6607225]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.213831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9be2593-3927-4a98-a865-80e44ebe0635]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.257000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b66048c-5498-456b-8129-ad75732fb08b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 126938, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.287567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f8391c3-df95-4a60-8d0e-4b034b28bb52]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.324761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73c902e0-1243-4d74-960f-7ed3b4d1a73f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.352783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41af7e61-6009-4054-8a1e-f4bdc282c5e6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.388063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c7c600f-a516-4adf-8bf5-55d61d263a64]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.389019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:22.390157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:22.390833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:22.391406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:22.440615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:22.445838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0891df3b-2221-4654-b1bf-c2789d7b92f2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.451230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40d0e02b-8194-403c-aed2-180217537d76]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.452734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:22.505695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:22.507704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:22.512008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:22.515751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:22.516488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:22.587110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a7b6bb0-e7b4-467a-ad2b-ece22a81fcca]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.648203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5bcbf263-2fbd-491d-bb9c-394d6fdf133a]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:22.648203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:22.648203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:22.648203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:22.648203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:22.648203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:22.648203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:22.648203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:22.648203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:22.648203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:22.648203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:22.648203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:22.648203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:22.649343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bcbf263-2fbd-491d-bb9c-394d6fdf133a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:22.650300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:22.652026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:22.654096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c96e71d3-4644-4717-baac-47cb970b5e04]: (4, ['qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.654096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:22.654096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:22.654096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:22.654096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:22.655080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e7c5df5-44d0-4edd-8116-e55eac9baa3c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.657545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8babaf0a-eeb7-4db4-b447-fa56eb9a2a32]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.658556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d88c40c-6ea0-4b4f-a6c1-66f35e4dbd03]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.709926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[062cb68f-e85f-417e-b46d-a58c2b0db504]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 253852, 'reachable_time': 28172, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 126964, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.720635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:22.721468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cccc9ed0-1848-4569-a090-dfb9659340f1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:22.724627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ff4cb520-36eb-4815-860e-ab86ac8aecec", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.192", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "6612ab5f-3352-4254-8460-dac56322c900", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 525, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:d8:4f", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "672c983b-c364-46e8-90d8-0553b70b3325", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:13Z", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:02Z", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:22.727571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:18Z", Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 21799d67-90be-4540-8039-89081fc0472b. Wait time elapsed: 17.721 Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 21799d67-90be-4540-8039-89081fc0472b. Wait time elapsed: 17.721 Jul 01 09:27:22.745879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:24.234013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.507s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:24.234651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:24.235239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:24.236044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:24.236325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:24.237467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43b10390-ec06-422a-9d05-b9ce86e5130f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.266008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea4afe25-2e77-49d2-9d74-56b61c54cd3d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.305968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-5e7616aa-e4d6-420e-a679-614da00fef3d tempest-RoutersAdminTest-1423367726 tempest-RoutersAdminTest-1423367726-project] Got routers updated notification :['95489426-ac13-45b2-a6e3-37c345fbb255'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:24.309335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[162a461a-0aed-445d-b92d-09d42d136cfc]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.310643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 95489426-ac13-45b2-a6e3-37c345fbb255, action 3, priority 1, update_id dcdd62a5-5eac-4355-9805-93a9c4eb1e63. Wait time elapsed: 0.003 Jul 01 09:27:24.311063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 95489426-ac13-45b2-a6e3-37c345fbb255, action 3, priority 1, update_id dcdd62a5-5eac-4355-9805-93a9c4eb1e63. Wait time elapsed: 0.004 Jul 01 09:27:24.311544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:24.316318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.982s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:24.317207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:24.318920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:24.329481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:24.330027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.011s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:24.332681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cae7bd87-6fc6-406d-9425-dc6786b64527]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.375583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3366395f-265a-4521-85dd-e6a28e2292ed]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.386965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8657a50c-c980-4822-8652-aac9bb6528c7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 127036, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.411672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8c47887-2c33-4a98-aa1b-37024f91feb5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.414841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f841f1a-6cf3-4b41-bfe3-a5797a7764c3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.451900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1cfc90d-b528-40af-9c8a-8a0bcb90dc05]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.466943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[046d9465-2037-4126-b7f3-f002da83ec37]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 127041, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.490556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14262d6c-000c-4369-a32c-fb2432799317]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.499009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04857f27-cec6-4a41-9124-35d5e6379b6a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.528297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6752d956-68b8-4f00-aa11-585de2f4f197]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.529177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:24.529612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:24.530032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:24.530502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:24.539216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16304d2b-5223-4817-a092-9ff3f275c181]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.564920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:24.566500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b68b72a1-37de-4849-90a1-7142200d5081]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.570385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a16544e4-99d8-4d86-992a-3c21f13c1f2d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.571900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:24.594134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e03d9ef1-70c9-46cd-aecb-f1585ecfcb3d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.612024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:24.617249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:24.619076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:24.620259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:24.620974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:24.681189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df6c1d4b-fe64-4280-ab1f-539f17132671]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.682751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:24.683174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:24.684791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:24.685082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:24.694362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7351a718-1c31-41e7-8cf1-1b2170713f1b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.710133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:24.710354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0cb9963-b476-462b-85f6-7a12406aaa1d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.714318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c92ed1e6-38c1-4874-b581-4ff1c116d432]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.715465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:24.750329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:24.751435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:24.752325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:24.752930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:24.753399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[952ea048-b1b3-4226-8cef-ea2043e2b336]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:24.755432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[952ea048-b1b3-4226-8cef-ea2043e2b336]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:24.760353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:24.764175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:24.788882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0cc104a1-93bc-4e02-8c12-509b4436cc83]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.790918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:24.793722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:24.794052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:24.794539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:24.797499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6dc99b1a-63a0-420f-8425-ffb7249e56ea]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.798998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[093206db-67c4-4bb9-a92a-19a0ceaede5c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.811062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c529e61-d4e3-4676-9245-3bfd8a921f1e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.813856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed1aa06a-f0ba-4cb9-b0ab-3b457cc2f221]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.841144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a59f4849-707a-4343-8810-4f6175f2db04]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 254064, 'reachable_time': 36224, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127074, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.843922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:24.844392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d6e258c-ea2e-497c-b7d7-3a33742b4210]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:24.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ff4cb520-36eb-4815-860e-ab86ac8aecec", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.192", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "6612ab5f-3352-4254-8460-dac56322c900", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 525, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:d8:4f", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "672c983b-c364-46e8-90d8-0553b70b3325", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:13Z", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:02Z", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:24.851917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:22Z", Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 21799d67-90be-4540-8039-89081fc0472b. Wait time elapsed: 19.853 Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 21799d67-90be-4540-8039-89081fc0472b. Wait time elapsed: 19.854 Jul 01 09:27:24.871544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:24.886103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[af5539f9-f232-4729-b151-5df0c04ccc6b]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:24.886103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:24.886103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:24.886103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:24.886103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:24.886103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:24.886103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:24.886103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:24.886103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:24.886103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:24.886103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:24.886103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:24.886103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:24.887168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af5539f9-f232-4729-b151-5df0c04ccc6b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:24.887911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:24.889860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:24.892284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0497ce1-5c09-4de3-9d28-fc7354ef27ca]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.893036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:24.894024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:24.894024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:24.894024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:24.897216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eba43e13-3609-4683-beff-8a38c074a8cb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.898386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25f48861-a94d-42ca-832e-0c91cbbe2d1f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.899660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55c304a4-7e4d-446a-8133-909419be350c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.929842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce5d7e3a-b8ff-40e5-bd81-317681226c29]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 254072, 'reachable_time': 27244, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127077, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.937263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:24.937744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6196bda4-30ac-48fb-8790-6df0e07e76da]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:24.938955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8e7d7bd8-acc2-4356-b50e-ff776e5c8d67", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.40", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "20d9e825-4027-4a35-987e-9fbcec875296", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.12", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 522, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:43:53:56", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4736091a-3a94-4af2-9e3d-4d661a5d3f11", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:12Z", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:01Z", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:24.944153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:20Z", Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id f1931dc6-0f2a-4abc-9a94-001f8baec952. Wait time elapsed: 20.201 Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id f1931dc6-0f2a-4abc-9a94-001f8baec952. Wait time elapsed: 20.205 Jul 01 09:27:24.960502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:25.758465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.897s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:25.759097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:25.760282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:25.760778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:25.763025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:25.765384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e79dc35a-11b0-44d0-b918-780ecbb77835]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:25.782341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.471s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:25.783121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 95489426-ac13-45b2-a6e3-37c345fbb255 was not found. Performing router cleanup Jul 01 09:27:25.785639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[919233fa-bf0f-432c-be3e-f4cd60a8e375]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:25.786725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:25.787194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:25.787619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 95489426-ac13-45b2-a6e3-37c345fbb255, update_id dcdd62a5-5eac-4355-9805-93a9c4eb1e63. Time elapsed: 1.478 Jul 01 09:27:25.795267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04d4ad4f-c470-4f7e-9840-f85c5a81430e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:25.830476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c53aabe-2d95-4075-b8ae-503302992178]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:25.889057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b8a0f24-5823-4375-93a1-19ff07d27a54]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 127080, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:25.924397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed311e6f-3fff-4219-b0e3-0f4b4881e693]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:25.960302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f60dadf8-5c5c-4a7a-a46f-106556a7e076]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:25.986433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d857f9d6-4e69-4be9-9d78-3102f17fe0b3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:26.035620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b753af14-1b53-4d71-a030-012796e253c1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:26.036357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:26.039144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:26.039657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:26.040902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:26.067433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:26.068037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39f2da75-869f-4fcb-872e-7b1976f82eb3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:26.071791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8532295c-a435-4244-b443-7c5d965fc283]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:26.073054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:26.118525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:26.119555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:26.120146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:26.120730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:26.121338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:26.177491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24581dc1-25ed-4a07-8ce5-70e05d74412c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:26.219327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-5b95d14d-ff2b-4582-aaf2-ff997c78c96f tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:26.220538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, action 3, priority 1, update_id 012898ff-e386-4e42-9685-9000e2e0a535. Wait time elapsed: 0.001 Jul 01 09:27:26.221116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, action 3, priority 1, update_id 012898ff-e386-4e42-9685-9000e2e0a535. Wait time elapsed: 0.001 Jul 01 09:27:26.221551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:26.234623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e27adbf5-6b75-45fe-b16f-b7062cb3a45e]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:26.234623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:26.234623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:26.234623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:26.234623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:26.234623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:26.234623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:26.234623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:26.234623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:26.234623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:26.234623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:26.234623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:26.234623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:26.236153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e27adbf5-6b75-45fe-b16f-b7062cb3a45e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:26.240823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:26.244382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:26.248228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[740f1880-e7a4-4693-9a70-83f7848a3c16]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:26.251564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:26.253390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:26.253390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:26.253390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:26.255330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f802739-20ff-403c-a46b-8e81f4e4c4bd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:26.257642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2db335a-ab53-47e9-b440-a01742573332]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:26.258971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7da20df5-4a6c-4146-888d-e6e045fefacc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:26.293967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbee3050-bece-48cd-88cc-7d29eec20eac]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 254214, 'reachable_time': 41224, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127099, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:26.303467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:26.303890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bed95285-827a-4141-a1f0-c50ae0f267ca]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:27:26.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ff4cb520-36eb-4815-860e-ab86ac8aecec", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.192", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "6612ab5f-3352-4254-8460-dac56322c900", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 525, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:d8:4f", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "672c983b-c364-46e8-90d8-0553b70b3325", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:13Z", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:02Z", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:26.309364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:22Z", Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:26.323723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3 Jul 01 09:27:26.409664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.188s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:26.409975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97 was not found. Performing router cleanup Jul 01 09:27:26.411065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f1ae546-022d-4983-b1f4-6b0617865972]: (4, ['qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:26.411950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:26.412292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:26.412527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, update_id 012898ff-e386-4e42-9685-9000e2e0a535. Time elapsed: 0.192 Jul 01 09:27:26.686132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-0b7b9279-5e7b-470f-a77b-3a470a786cb9 tempest-RoutersNegativeTest-1826052474 tempest-RoutersNegativeTest-1826052474-project] Got routers updated notification :['3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:26.688301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, action 3, priority 1, update_id c7122fea-b430-4095-887e-de01e81ca63f. Wait time elapsed: 0.001 Jul 01 09:27:26.688782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, action 3, priority 1, update_id c7122fea-b430-4095-887e-de01e81ca63f. Wait time elapsed: 0.001 Jul 01 09:27:26.689280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:27.176654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-554527ef-291d-40e2-9667-89c99752d37f tempest-RoutersAdminTest-1423367726 tempest-RoutersAdminTest-1423367726-project] Got router deleted notification for 95489426-ac13-45b2-a6e3-37c345fbb255 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:27:27.178009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 95489426-ac13-45b2-a6e3-37c345fbb255, action 1, priority 1, update_id 2cf0c9fe-6329-4e69-b13a-947db08b9e39. Wait time elapsed: 0.001 Jul 01 09:27:27.178009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 95489426-ac13-45b2-a6e3-37c345fbb255, action 1, priority 1, update_id 2cf0c9fe-6329-4e69-b13a-947db08b9e39. Wait time elapsed: 0.001 Jul 01 09:27:27.178208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 95489426-ac13-45b2-a6e3-37c345fbb255 was not found. Performing router cleanup Jul 01 09:27:27.179348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[005fa449-58a6-4211-b044-3a0fea41e094]: (4, ['qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:27.180215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:27.180484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:27.180737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 95489426-ac13-45b2-a6e3-37c345fbb255, update_id 2cf0c9fe-6329-4e69-b13a-947db08b9e39. Time elapsed: 0.003 Jul 01 09:27:27.392613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.703s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:27.394178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d was not found. Performing router cleanup Jul 01 09:27:27.406687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46191baf-038e-4c04-a2ca-e22baa01b619]: (4, ['qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:27.407255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:27.407572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:27.407887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, update_id c7122fea-b430-4095-887e-de01e81ca63f. Time elapsed: 0.720 Jul 01 09:27:29.549769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.589s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:29.551828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:29.553567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:29.554097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:29.554570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:29.556181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c795fdba-58f9-4f5f-a147-593d736fa0b6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:29.583232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b9adbb2-62ac-47fa-8f03-beeda75c7a18]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:29.619030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1fa2f41-afa1-4819-bab0-ac8734f3974f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:29.666508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[296796f6-5bfe-4c67-856d-9edfad12bd0c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 127104, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:29.698236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d7397e2-4a22-4441-9d42-7e47d36f3720]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:29.725371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f537839e-4350-4883-9693-003117c515fd]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:29.749528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[316f9062-6482-45eb-8f6a-fe5845887bad]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:29.779068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[778c7829-bbda-4255-9b1e-d9db09dd76b7]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:29.779738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:29.780015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:29.780279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:29.780557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:29.819911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:29.822858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d98d84e-3b12-4f05-b05f-18becca62142]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:29.841003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90e72723-859d-4c15-bce8-bf5265f2b9e4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:29.841003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:29.890557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:29.892367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:29.895968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:29.896650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:29.897226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:29.959877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8ba9e4b-0144-4437-a7bb-3c29ca31a6e2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:30.001725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3a291e9e-1cf8-4dd7-bc08-c0f83411375d]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:30.001725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:30.001725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:30.001725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:30.001725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:30.001725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:30.001725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:30.001725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:30.001725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:30.001725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:30.001725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:30.001725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:30.001725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:30.002660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a291e9e-1cf8-4dd7-bc08-c0f83411375d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:30.002871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:30.006452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:30.010764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6a26643-b8bf-4a51-9c09-bd2f84159b55]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:30.011744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:30.012253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:30.012676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:30.013198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:30.014479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14c8f9b1-ac14-405a-8cf9-afd6b346afc2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:30.015792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8e4cd92-4c6c-4b7f-9c39-66d4d7b91891]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:30.016906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be9678c8-b6a9-48d5-a861-26416e76f1e1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:30.037246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abc8f209-16bf-4494-8ee0-60497c73f292]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 254592, 'reachable_time': 26500, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127138, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:30.046507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:30.046972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee2d9c4f-9fed-4135-843e-6feaf8292abd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:27:30.048210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8e7d7bd8-acc2-4356-b50e-ff776e5c8d67", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.40", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "20d9e825-4027-4a35-987e-9fbcec875296", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.12", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 522, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:43:53:56", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4736091a-3a94-4af2-9e3d-4d661a5d3f11", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:12Z", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:01Z", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:30.053316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:27Z", Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:30.062874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:30.069113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3 Jul 01 09:27:30.664975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-01754782-c3c8-4fcb-a37a-aef74525044f tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:30.665921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, action 3, priority 1, update_id 82d010ad-203b-4f6b-83e9-fa2977b1690b. Wait time elapsed: 0.001 Jul 01 09:27:30.666391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, action 3, priority 1, update_id 82d010ad-203b-4f6b-83e9-fa2977b1690b. Wait time elapsed: 0.001 Jul 01 09:27:30.667010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:30.850577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.183s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:30.850842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97 was not found. Performing router cleanup Jul 01 09:27:30.851908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bb96bd7-d478-4eb1-85a0-4a5f9b449cb2]: (4, ['qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:30.852383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:30.852649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:30.852895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, update_id 82d010ad-203b-4f6b-83e9-fa2977b1690b. Time elapsed: 0.187 Jul 01 09:27:32.958772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-1e733185-a20e-4529-8976-f9e2cdc26b87 tempest-NetworksTestDHCPv6-1249225972 tempest-NetworksTestDHCPv6-1249225972-project] Got routers updated notification :['873949ec-0fbf-48fb-ba35-1c45487cb73e'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:32.965067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 873949ec-0fbf-48fb-ba35-1c45487cb73e, action 3, priority 1, update_id dc6a6e73-cdbc-4ed7-9371-0db97dce41c5. Wait time elapsed: 0.001 Jul 01 09:27:32.965604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 873949ec-0fbf-48fb-ba35-1c45487cb73e, action 3, priority 1, update_id dc6a6e73-cdbc-4ed7-9371-0db97dce41c5. Wait time elapsed: 0.002 Jul 01 09:27:32.966603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:34.137491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-a941858b-6576-485c-baef-0e0af77aeefb tempest-RoutersAdminTest-1421187323 tempest-RoutersAdminTest-1421187323-project-admin] Got routers updated notification :['9267978f-7dc4-458f-a22a-e9d7585ead0d'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:34.140871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9267978f-7dc4-458f-a22a-e9d7585ead0d, action 3, priority 1, update_id ee6ea5f5-ad90-497d-84f9-98b8eb716cf8. Wait time elapsed: 0.002 Jul 01 09:27:34.141415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9267978f-7dc4-458f-a22a-e9d7585ead0d, action 3, priority 1, update_id ee6ea5f5-ad90-497d-84f9-98b8eb716cf8. Wait time elapsed: 0.003 Jul 01 09:27:34.142613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:34.404373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-fdb8abb9-0bad-400b-98ec-8096b19f0b3d tempest-RoutersNegativeTest-1826052474 tempest-RoutersNegativeTest-1826052474-project] Got routers updated notification :['3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:34.405323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, action 3, priority 1, update_id bd316ac2-40d1-4298-84c0-73fc6038dcae. Wait time elapsed: 0.001 Jul 01 09:27:34.405579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, action 3, priority 1, update_id bd316ac2-40d1-4298-84c0-73fc6038dcae. Wait time elapsed: 0.001 Jul 01 09:27:34.406062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:34.447994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.305s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:34.448261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9267978f-7dc4-458f-a22a-e9d7585ead0d was not found. Performing router cleanup Jul 01 09:27:34.449688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2024f386-45af-456d-9dc2-1b2ff54640a5]: (4, ['qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:34.450377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:34.450650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:34.451102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9267978f-7dc4-458f-a22a-e9d7585ead0d, update_id ee6ea5f5-ad90-497d-84f9-98b8eb716cf8. Time elapsed: 0.311 Jul 01 09:27:34.850225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.444s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:34.850485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d was not found. Performing router cleanup Jul 01 09:27:34.852681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4da9cc56-a5fc-49d2-9ee3-b9ed9404d926]: (4, ['qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:34.854264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:34.855018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:34.855451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, update_id bd316ac2-40d1-4298-84c0-73fc6038dcae. Time elapsed: 0.450 Jul 01 09:27:35.072908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:27:35.509059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-0f3d77e0-8a2c-4dbc-838f-14998490fa12 tempest-NetworksTestDHCPv6-1249225972 tempest-NetworksTestDHCPv6-1249225972-project] Got routers updated notification :['873949ec-0fbf-48fb-ba35-1c45487cb73e'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:36.042659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-13cc3472-892e-484f-9e1f-b467c1502c8f tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:36.044524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, action 3, priority 1, update_id 0b8e3471-6bde-4f8f-a9eb-a2bfb6cec980. Wait time elapsed: 0.001 Jul 01 09:27:36.044990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, action 3, priority 1, update_id 0b8e3471-6bde-4f8f-a9eb-a2bfb6cec980. Wait time elapsed: 0.002 Jul 01 09:27:36.046471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:36.226449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.180s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:36.228074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97 was not found. Performing router cleanup Jul 01 09:27:36.230668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba98d1ec-eb27-43e3-8532-edcd921ffc3a]: (4, ['qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:36.231708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:36.232327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:36.232859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, update_id 0b8e3471-6bde-4f8f-a9eb-a2bfb6cec980. Time elapsed: 0.188 Jul 01 09:27:36.399337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.432s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:36.399679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 873949ec-0fbf-48fb-ba35-1c45487cb73e info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:36.400820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (873949ec-0fbf-48fb-ba35-1c45487cb73e), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:36.401144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:36.401433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:36.402924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92155f1f-c6bc-46a0-8068-b0839f47012c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:36.437690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[095481e9-924d-421a-ab64-71f0dd31f8a1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:36.474880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a59ef7f8-c2dc-4839-9c9c-1640540300ee]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:36.561063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45d7cde9-4239-4583-96b4-82c71a82d6e6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 127237, 'error': None, 'target': 'qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:36.598243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23c3ab6e-255d-4df5-93bf-2ba047726e06]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:36.642701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2fd37bd-c771-4b34-b66e-a02a6cc6e06a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:36.683445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59c46dc7-0376-439c-9cf0-84396d8b2e44]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:36.712522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b562931-f877-4208-90a5-4b7cdd23e183]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:36.715675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:36.716341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:36.716928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 873949ec-0fbf-48fb-ba35-1c45487cb73e {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:36.725314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-edbdddd7-c88f-4120-9f09-b1cf6416ab56 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got router deleted notification for 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:27:36.727454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, action 1, priority 1, update_id 02812a2b-2d6f-4b5f-a125-c2b2c11f776b. Wait time elapsed: 0.001 Jul 01 09:27:36.727967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, action 1, priority 1, update_id 02812a2b-2d6f-4b5f-a125-c2b2c11f776b. Wait time elapsed: 0.002 Jul 01 09:27:36.728494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97 was not found. Performing router cleanup Jul 01 09:27:36.730332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a1ee7fa-3f11-4198-940f-6603c657e8ae]: (4, ['qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:36.731661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:36.733452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:36.733452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 00a3f8e4-6c4e-4cbf-9549-3b1337ca8b97, update_id 02812a2b-2d6f-4b5f-a125-c2b2c11f776b. Time elapsed: 0.005 Jul 01 09:27:36.748719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73f40945-ff7e-4b82-92ae-3c7dcfd4a093]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 255280, 'reachable_time': 19580, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127243, 'error': None, 'target': 'qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:36.749542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:36.749824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:36.750248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:36.750508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:27:36.784745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4354ea5-2ebd-4100-8f49-4bef86bc70d7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 255280, 'reachable_time': 19580, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127244, 'error': None, 'target': 'qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:36.787808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:36.788172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:27:36.815901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8423324-20c5-446f-8adc-89000038a989]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:37.029987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0069ef9f-81b4-4541-a31e-19c2763575a7]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:37.065503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00bd2bfb-15ee-4a08-be8f-d8a4bbd850d6]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:37.295158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9c2de67-9ca4-4292-a2fb-86e74162033b]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:37.295969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:27:37.296277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:37.296603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:37.296884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" released by "process_external" :: held 0.546s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:27:37.297321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:37.297618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:27:37.297990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:37.298424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:27:37.313851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b9188bd2-2018-4390-add4-00fa3495e44e tempest-NetworksTestDHCPv6-1249225972 tempest-NetworksTestDHCPv6-1249225972-project] Got router deleted notification for 873949ec-0fbf-48fb-ba35-1c45487cb73e {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:27:37.357504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3d24377-7e32-4743-ad6b-208a11c078c8]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:37.402660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42516e87-45f0-47f9-bf81-d32ef447c7db]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:37.449444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d3d0494-2642-425f-a3b1-1cec9612ab58]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:37.530016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fad0568-0e6d-4ec7-8a8c-6e98f8180a1c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:37.530783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:27:37.531176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:37.531526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:37.531823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" released by "process_address_scope" :: held 0.234s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:27:37.532192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (873949ec-0fbf-48fb-ba35-1c45487cb73e), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:37.532654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:37.532900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:27:37.533400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:37.533856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:27:37.579206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3d33e96-9fc7-47eb-8356-6b58397b9005]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:37.657891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87db194a-01de-48cb-86f0-1770f8fbe539]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:37.716759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae9b350b-cfde-4c18-a2d3-26157761fc3c]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:37 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:27:37 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:37.821570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15fc3fea-8db4-4af9-9e60-38809a5590dd]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:37.822272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:27:37.822614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:37.822915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:37.824676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e" released by "apply_metadata_nat_rules" :: held 0.290s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:27:37.825446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/873949ec-0fbf-48fb-ba35-1c45487cb73e.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/873949ec-0fbf-48fb-ba35-1c45487cb73e.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-873949ec-0fbf-48fb-ba35-1c45487cb73e Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/873949ec-0fbf-48fb-ba35-1c45487cb73e.pid.haproxy Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 873949ec-0fbf-48fb-ba35-1c45487cb73e Jul 01 09:27:37.826075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:27:37.828264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/873949ec-0fbf-48fb-ba35-1c45487cb73e.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:27:37.875847 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-873949ec-0fbf-48fb-ba35-1c45487cb73e[127260]: Proxy listener started. Jul 01 09:27:37.892538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 873949ec-0fbf-48fb-ba35-1c45487cb73e add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:27:37.892798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 873949ec-0fbf-48fb-ba35-1c45487cb73e, update_id dc6a6e73-cdbc-4ed7-9371-0db97dce41c5. Time elapsed: 4.928 Jul 01 09:27:37.893239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 873949ec-0fbf-48fb-ba35-1c45487cb73e, action 3, priority 1, update_id 8f257a0b-2b6a-4935-9f32-197044760493. Wait time elapsed: 2.384 Jul 01 09:27:37.893633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 873949ec-0fbf-48fb-ba35-1c45487cb73e, action 3, priority 1, update_id 8f257a0b-2b6a-4935-9f32-197044760493. Wait time elapsed: 2.384 Jul 01 09:27:37.894075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:38.205976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.312s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:38.206348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 873949ec-0fbf-48fb-ba35-1c45487cb73e delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:27:38.206673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (873949ec-0fbf-48fb-ba35-1c45487cb73e), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:38.251347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5e56902-dc73-40dc-920e-8eb7a7bfaa95]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:38.253246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27dc1808-80a5-4ee6-b754-bb968dfe687d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:38.255022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f8c3dd5-285e-4c2c-85d4-984b8c2d9ab9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:38.256632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 873949ec-0fbf-48fb-ba35-1c45487cb73e {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:27:38.257813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5b0f91a-4da0-4e24-a092-245f701d0f01]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:38.309836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7600939b-793d-49c2-b607-3a12043c021f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 255280, 'reachable_time': 19580, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127266, 'error': None, 'target': 'qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:38.310579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:38.310851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:38.346270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2032189-cbca-48c9-af06-13face4650ec]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 255280, 'reachable_time': 19580, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127267, 'error': None, 'target': 'qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:38.348533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 873949ec-0fbf-48fb-ba35-1c45487cb73e {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:27:38.349453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/873949ec-0fbf-48fb-ba35-1c45487cb73e.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/873949ec-0fbf-48fb-ba35-1c45487cb73e.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:38.349992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/873949ec-0fbf-48fb-ba35-1c45487cb73e.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/873949ec-0fbf-48fb-ba35-1c45487cb73e.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:38.350420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 873949ec-0fbf-48fb-ba35-1c45487cb73e {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:38.351704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 873949ec-0fbf-48fb-ba35-1c45487cb73e {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:27:38.353093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92a578cb-8607-4fdb-9ac7-6c92511203da]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:38.372470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e248f98-39f4-423d-b032-a57ecc2f186d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 255280, 'reachable_time': 19580, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127268, 'error': None, 'target': 'qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:38.378612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-873949ec-0fbf-48fb-ba35-1c45487cb73e deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:38.379255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cfdfbc66-e827-4051-b0e9-04a3b315e851]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:38.380092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 873949ec-0fbf-48fb-ba35-1c45487cb73e delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:27:38.380645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (873949ec-0fbf-48fb-ba35-1c45487cb73e), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:38.381312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:38.381832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:38.382337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:38.382813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:38.383280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 873949ec-0fbf-48fb-ba35-1c45487cb73e, update_id 8f257a0b-2b6a-4935-9f32-197044760493. Time elapsed: 0.490 Jul 01 09:27:38.696491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-04426bf0-b848-4965-8521-17e5a2ea0674 tempest-RoutersAdminTest-1423367726 tempest-RoutersAdminTest-1423367726-project] Got router deleted notification for 9267978f-7dc4-458f-a22a-e9d7585ead0d {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:27:38.697837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9267978f-7dc4-458f-a22a-e9d7585ead0d, action 1, priority 1, update_id 358f9ea4-9692-4c37-b15f-01c3b87ba01a. Wait time elapsed: 0.001 Jul 01 09:27:38.698189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9267978f-7dc4-458f-a22a-e9d7585ead0d, action 1, priority 1, update_id 358f9ea4-9692-4c37-b15f-01c3b87ba01a. Wait time elapsed: 0.001 Jul 01 09:27:38.699616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9267978f-7dc4-458f-a22a-e9d7585ead0d was not found. Performing router cleanup Jul 01 09:27:38.701576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[740d7acc-aa56-4197-ab8a-1490337248df]: (4, ['qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:38.702252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:38.703056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:38.703465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9267978f-7dc4-458f-a22a-e9d7585ead0d, update_id 358f9ea4-9692-4c37-b15f-01c3b87ba01a. Time elapsed: 0.006 Jul 01 09:27:41.528705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-9d4b521e-46eb-4a3d-bf8d-cb71bddd4fa2 tempest-RoutersNegativeTest-1826052474 tempest-RoutersNegativeTest-1826052474-project] Got routers updated notification :['3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:41.531707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, action 3, priority 1, update_id 3181c406-abee-4c7c-a7bd-e34723c9963c. Wait time elapsed: 0.002 Jul 01 09:27:41.532249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, action 3, priority 1, update_id 3181c406-abee-4c7c-a7bd-e34723c9963c. Wait time elapsed: 0.002 Jul 01 09:27:41.532562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:41.666490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.134s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:41.666797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d was not found. Performing router cleanup Jul 01 09:27:41.668062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8ace0cf-0dac-4870-9c9f-0a234ef4154e]: (4, ['qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:41.668536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:41.668799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:41.669048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, update_id 3181c406-abee-4c7c-a7bd-e34723c9963c. Time elapsed: 0.137 Jul 01 09:27:43.403797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-7004340a-cd12-4d4d-adfc-441ecc79c754 tempest-RoutersAdminTest-1421187323 tempest-RoutersAdminTest-1421187323-project-admin] Got routers updated notification :['18317daa-309a-4a36-b835-64cc148fc283'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:43.405706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 18317daa-309a-4a36-b835-64cc148fc283, action 3, priority 1, update_id 055171b0-f209-41f8-a8b2-a80148c31880. Wait time elapsed: 0.001 Jul 01 09:27:43.407628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 18317daa-309a-4a36-b835-64cc148fc283, action 3, priority 1, update_id 055171b0-f209-41f8-a8b2-a80148c31880. Wait time elapsed: 0.001 Jul 01 09:27:43.408146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:43.968329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.560s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:43.968329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 18317daa-309a-4a36-b835-64cc148fc283 was not found. Performing router cleanup Jul 01 09:27:43.968329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85588964-11df-466c-87f3-ed1543334ef7]: (4, ['qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:43.968329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:43.968329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:43.968329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 18317daa-309a-4a36-b835-64cc148fc283, update_id 055171b0-f209-41f8-a8b2-a80148c31880. Time elapsed: 0.565 Jul 01 09:27:44.118239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-978ef4af-fc66-483f-af8a-d1057e5c39ac tempest-NetworksTestDHCPv6-1249225972 tempest-NetworksTestDHCPv6-1249225972-project] Got routers updated notification :['1764c0b6-02c3-41fb-89a4-7707b146c7b1'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:44.121187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 1764c0b6-02c3-41fb-89a4-7707b146c7b1, action 3, priority 1, update_id 5dc30a7d-2f3a-46d6-a05e-9fd2b40ab07e. Wait time elapsed: 0.001 Jul 01 09:27:44.121671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 1764c0b6-02c3-41fb-89a4-7707b146c7b1, action 3, priority 1, update_id 5dc30a7d-2f3a-46d6-a05e-9fd2b40ab07e. Wait time elapsed: 0.003 Jul 01 09:27:44.123037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:46.400806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.278s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:46.401501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 1764c0b6-02c3-41fb-89a4-7707b146c7b1 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:46.402127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (1764c0b6-02c3-41fb-89a4-7707b146c7b1), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:46.402636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:46.403129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:46.404511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb82a8a1-bc46-4ec6-b2d7-b01ad7dd9110]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.439475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3edd9ac-fe52-43fc-8e18-df4adc3e3ebe]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.468035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1bf5461f-88ac-431d-bbea-70477110371b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.509781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-0d5f5128-41d6-4662-a480-56bf56dd861c tempest-RoutersNegativeTest-1826052474 tempest-RoutersNegativeTest-1826052474-project] Got routers updated notification :['3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:46.511007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, action 3, priority 1, update_id 9ad51eda-52f0-4e0b-a7d7-5f13f7663f6c. Wait time elapsed: 0.001 Jul 01 09:27:46.511905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, action 3, priority 1, update_id 9ad51eda-52f0-4e0b-a7d7-5f13f7663f6c. Wait time elapsed: 0.002 Jul 01 09:27:46.512425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:46.532531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9234675c-5a45-4485-9b4c-91e78afbb447]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 127402, 'error': None, 'target': 'qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.571138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85d2c8a1-c619-41f9-aa15-b75da3447c7f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.611199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51449946-6d98-4362-9641-a14bd2dc20f1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.648704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e32ec0e-187c-4ab7-afd0-a7352dc128f8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.684758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4982a7ba-096f-41aa-901a-bca717127596]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.685418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:46.685814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:46.686735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 1764c0b6-02c3-41fb-89a4-7707b146c7b1 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:46.687239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(5c498fad-d8eb-4554-8f9a-a8d48f071d05) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:46.713294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:46.713575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4852f3a2-fd01-42b8-8de6-f4c480881329]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.718398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23b99bee-2de9-4c66-b3d4-79e5c2b64327]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.720472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-5c498fad-d8, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:46.720814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:27:46.721864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-5c498fad-d8, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:46.723602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-5c498fad-d8, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:46.723809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-5c498fad-d8, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:46.724313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-5c498fad-d8, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:46.724800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-5c498fad-d8, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '5c498fad-d8eb-4554-8f9a-a8d48f071d05', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:e0:65:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:46.753577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d232e341-feb4-4dc9-b34e-45cb60ac865b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.814657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ff1c36ca-1181-4dfe-997d-5b860093bf59]: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:46.814657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:46.814657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:46.814657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:46.814657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:46.814657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:46.814657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:46.814657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:46.814657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:46.814657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:46.814657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:46.814657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:46.814657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:46.815715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff1c36ca-1181-4dfe-997d-5b860093bf59]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:46.816055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 1764c0b6-02c3-41fb-89a4-7707b146c7b1: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:46.818118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:46.826478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9282928-bfab-4cac-88ac-1dc3fbcf9721]: (4, ['qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.827466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:46.827678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:46.828856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 1764c0b6-02c3-41fb-89a4-7707b146c7b1 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:46.829205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:46.830326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[214e894d-465c-4769-8734-0a6311ffc98f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.831224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cdedc742-0466-4784-aed4-5fdadba7b312]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.832174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f58b7e49-064a-46e1-8b39-0651a63a6d3b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.873459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47091fdb-6d13-4a98-83fa-004d944da2be]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 256279, 'reachable_time': 32792, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127419, 'error': None, 'target': 'qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.877807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:46.877807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf62fb52-c41d-498c-8828-eaa0d069d6ab]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 1764c0b6-02c3-41fb-89a4-7707b146c7b1: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:46.879631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1764c0b6-02c3-41fb-89a4-7707b146c7b1", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-NetworksTestDHCPv6-router-1151292740", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "b0574abf371649318a1bae032a0da33e", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:27:38Z", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:42Z", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 2, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "b0574abf371649318a1bae032a0da33e", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5c498fad-d8eb-4554-8f9a-a8d48f071d05", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "d7a37a39-831d-48e1-a7e3-b045dceb482e", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "b0574abf371649318a1bae032a0da33e", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e0:65:69", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1764c0b6-02c3-41fb-89a4-7707b146c7b1", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 785, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "9f2feac4-2912-4b42-a5ad-4a857bab752e", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8::1", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:27:40Z", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:45Z", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "b0574abf371649318a1bae032a0da33e", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9f2feac4-2912-4b42-a5ad-4a857bab752e", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8::/64", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8::1", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": "dhcpv6-stateful", Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 1764c0b6-02c3-41fb-89a4-7707b146c7b1, action 3, priority 2, update_id 5dc30a7d-2f3a-46d6-a05e-9fd2b40ab07e. Wait time elapsed: 2.762 Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 1764c0b6-02c3-41fb-89a4-7707b146c7b1, action 3, priority 2, update_id 5dc30a7d-2f3a-46d6-a05e-9fd2b40ab07e. Wait time elapsed: 2.762 Jul 01 09:27:46.882637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:47.387956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.875s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:47.388210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d was not found. Performing router cleanup Jul 01 09:27:47.389283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48b6df41-b54c-4379-978a-88ee077a12b3]: (4, ['qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:47.390094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:47.390360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:47.390608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, update_id 9ad51eda-52f0-4e0b-a7d7-5f13f7663f6c. Time elapsed: 0.880 Jul 01 09:27:47.450604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-f7e1f7fa-07cb-4b7f-987f-294a1ba5f3c6 tempest-RoutersAdminTest-1423367726 tempest-RoutersAdminTest-1423367726-project] Got router deleted notification for 18317daa-309a-4a36-b835-64cc148fc283 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:27:47.451523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 18317daa-309a-4a36-b835-64cc148fc283, action 1, priority 1, update_id 1f65fa16-cac4-4ebd-ac3c-58350074c183. Wait time elapsed: 0.001 Jul 01 09:27:47.451767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 18317daa-309a-4a36-b835-64cc148fc283, action 1, priority 1, update_id 1f65fa16-cac4-4ebd-ac3c-58350074c183. Wait time elapsed: 0.001 Jul 01 09:27:47.452012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 18317daa-309a-4a36-b835-64cc148fc283 was not found. Performing router cleanup Jul 01 09:27:47.454203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e2e88f8-099c-4601-af09-a6e6a8466249]: (4, ['qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:47.455350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:47.455975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:47.456466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 18317daa-309a-4a36-b835-64cc148fc283, update_id 1f65fa16-cac4-4ebd-ac3c-58350074c183. Time elapsed: 0.005 Jul 01 09:27:47.892733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.012s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:47.893122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 1764c0b6-02c3-41fb-89a4-7707b146c7b1 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:47.894749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (1764c0b6-02c3-41fb-89a4-7707b146c7b1), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:47.895229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:47.895699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:47.897140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[652124ac-309d-405d-8a5c-35c9b39a07a0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:47.919452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[917573a7-cc8e-4fb5-b775-4719a7ca7ce7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:47.947752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4bff2dba-ede4-4c9b-8fa6-1b55ffc1dec2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:47.990352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca5479e5-ab95-4777-b816-6cbf94e690a1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 127423, 'error': None, 'target': 'qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.022003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25021fda-8aa4-47c0-a54c-0666164cd52d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.048654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a44b2ff-02ab-4a87-8c20-5f9b13bad547]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.071275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f014fc80-78a4-47b1-bf17-f1dbd15f0cbf]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.094072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62e7a2fe-a1f5-4b0b-8172-916c58f3a871]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.096009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:48.096534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:48.097029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 1764c0b6-02c3-41fb-89a4-7707b146c7b1 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:48.098129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(5c498fad-d8eb-4554-8f9a-a8d48f071d05) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:27:48.122223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:27:48.124267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e26fdcdb-4b29-4273-b18a-490cc4e8d7d1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.128828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f797075-fe96-4076-b3e8-3804a4a285fc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.130155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-5c498fad-d8, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:48.161850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-5c498fad-d8, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:48.162526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-5c498fad-d8, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:48.164027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-5c498fad-d8, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:48.165500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-5c498fad-d8, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:48.166029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-5c498fad-d8, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '5c498fad-d8eb-4554-8f9a-a8d48f071d05', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:e0:65:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:27:48.194238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4bdcc0b-dd76-464e-993f-1ba0fc8777cb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.231799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[71c47486-1b39-4343-882c-b19679d64206]: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:27:48.231799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:27:48.231799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:27:48.231799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:27:48.231799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:27:48.231799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:27:48.231799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:27:48.231799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:27:48.231799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:27:48.231799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:27:48.231799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:27:48.231799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:27:48.231799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:48.232735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71c47486-1b39-4343-882c-b19679d64206]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:48.233907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 1764c0b6-02c3-41fb-89a4-7707b146c7b1: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:48.236364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[121dfce3-3d14-44b8-a555-13708dac7a55]: (4, ['qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.238699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:48.238699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:48.238699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 1764c0b6-02c3-41fb-89a4-7707b146c7b1 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:48.238913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:48.240750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[482427f5-94e6-4084-a111-cc3c734b9b4c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.242190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2cf7408-4032-4e67-bf77-13c729df796e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.243532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd8774fd-51c6-4ee6-8e11-9e76e856f135]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.279124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c730f63-4844-48d3-aa05-73e44dbab665]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 256425, 'reachable_time': 44988, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127442, 'error': None, 'target': 'qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.287597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:48.288393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e3e17ab-1510-40c0-a4e4-dfa5b6bfb79d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 1764c0b6-02c3-41fb-89a4-7707b146c7b1: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5c498fad-d8 not found in namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1. Jul 01 09:27:48.289696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1764c0b6-02c3-41fb-89a4-7707b146c7b1", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-NetworksTestDHCPv6-router-1151292740", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "b0574abf371649318a1bae032a0da33e", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:27:38Z", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:42Z", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 2, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "b0574abf371649318a1bae032a0da33e", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5c498fad-d8eb-4554-8f9a-a8d48f071d05", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "d7a37a39-831d-48e1-a7e3-b045dceb482e", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "b0574abf371649318a1bae032a0da33e", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e0:65:69", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1764c0b6-02c3-41fb-89a4-7707b146c7b1", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 785, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "9f2feac4-2912-4b42-a5ad-4a857bab752e", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8::1", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:27:40Z", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:45Z", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "b0574abf371649318a1bae032a0da33e", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9f2feac4-2912-4b42-a5ad-4a857bab752e", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8::/64", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8::1", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": "dhcpv6-stateful", Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:27:48.295614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:27:48.308807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 1764c0b6-02c3-41fb-89a4-7707b146c7b1, action 3, priority 2, update_id 5dc30a7d-2f3a-46d6-a05e-9fd2b40ab07e. Wait time elapsed: 4.190 Jul 01 09:27:48.309059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 1764c0b6-02c3-41fb-89a4-7707b146c7b1, action 3, priority 2, update_id 5dc30a7d-2f3a-46d6-a05e-9fd2b40ab07e. Wait time elapsed: 4.190 Jul 01 09:27:48.310253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:49.626304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.316s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:49.628172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 1764c0b6-02c3-41fb-89a4-7707b146c7b1 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:27:49.630147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (1764c0b6-02c3-41fb-89a4-7707b146c7b1), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:49.630744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:49.631805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:49.633539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d616a48-c2fb-4753-ac87-7bccba037f97]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:49.662396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7016c8fe-2198-42a2-806a-fa07088b9406]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:49.694317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[174e667d-dd56-4c6c-8931-7a1748bbce50]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:49.747248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4ac990e-82ce-4f95-aa63-be35dec5a379]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 127446, 'error': None, 'target': 'qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:49.786697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2e3af85-a5b3-4c07-a81a-5a737a1759cb]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:49.822034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5060edf0-f633-4278-b0fe-943567d4ded5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:49.855253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d12d4f17-c5d6-49f4-8a32-bfa93766d6c8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:49.882522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8cf0c0bb-0a28-4017-afc6-e569a2c0dcf5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:49.884010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:49.884863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:49.885390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 1764c0b6-02c3-41fb-89a4-7707b146c7b1 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:49.913432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51d208a6-8cb3-411a-8277-8f48d4e0df8b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 256600, 'reachable_time': 43560, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127452, 'error': None, 'target': 'qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:49.914607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:49.914931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:49.915433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:49.916033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:27:49.944839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aba884bc-3885-4ffc-956e-bd43401a0620]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 256600, 'reachable_time': 43560, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127453, 'error': None, 'target': 'qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:49.946086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:49.946666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:27:49.973837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0dc5bf6-b937-4a0e-ae65-74ed1a7bbed3]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:50.061135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b9145566-6026-4c5e-9c0e-a4ee5c462061 tempest-NetworksTestDHCPv6-1249225972 tempest-NetworksTestDHCPv6-1249225972-project] Got routers updated notification :['1764c0b6-02c3-41fb-89a4-7707b146c7b1'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:50.160711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-25ef3dd0-bdd6-48bc-aee1-a993f4db656b tempest-RoutersAdminTest-1423367726 tempest-RoutersAdminTest-1423367726-project] Got routers updated notification :['d95961c7-a977-49e7-a8cd-81ed5941157a'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:50.164951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update d95961c7-a977-49e7-a8cd-81ed5941157a, action 3, priority 1, update_id ade8951a-f74c-4341-8696-082ecd52d010. Wait time elapsed: 0.002 Jul 01 09:27:50.167131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for d95961c7-a977-49e7-a8cd-81ed5941157a, action 3, priority 1, update_id ade8951a-f74c-4341-8696-082ecd52d010. Wait time elapsed: 0.003 Jul 01 09:27:50.169469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:50.206386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ab491fa-299c-4855-9cd2-65eca87520d7]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:50.262567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1107eb43-8444-4404-bacf-030ce8d736bf]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:50.442141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.273s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:50.442710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router d95961c7-a977-49e7-a8cd-81ed5941157a was not found. Performing router cleanup Jul 01 09:27:50.449516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b660701c-3bae-4033-a3f3-1770b820bd15]: (4, ['qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:50.453281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:50.453817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:50.454260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for d95961c7-a977-49e7-a8cd-81ed5941157a, update_id ade8951a-f74c-4341-8696-082ecd52d010. Time elapsed: 0.289 Jul 01 09:27:50.472516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bac1776-2369-4630-b6b0-0b3320c42946]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:50.473176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:27:50.477842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:50.479309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:50.480197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" released by "process_external" :: held 0.564s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:27:50.480842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:50.481788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" acquired by "process_address_scope" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:27:50.482330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:50.483064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:27:50.530227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51e6c947-83c4-4398-a3b9-961b3def0928]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:50.576116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20868884-6c2c-4716-804f-d5320fdb566b]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:50.617491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94868b76-46bc-4e7b-ac45-069e23b7c26e]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:50.669127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a3279fc-048c-411b-93a7-1e5112efcd27]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:50.671311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:27:50.672467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:50.672918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:50.673363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" released by "process_address_scope" :: held 0.192s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:27:50.673927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (1764c0b6-02c3-41fb-89a4-7707b146c7b1), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:50.675044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:50.675535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" acquired by "apply_metadata_nat_rules" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:27:50.676487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:50.677712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:27:50.703714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac318b8e-41b5-4ba0-a346-ebb39fb35a04]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:50.774976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c603b2ce-4f6d-4756-b8dc-05fe9ec74b96]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:50.820043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd44a39d-c6e7-44fb-9e7b-f312a079292f]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:50 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:27:50 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:50.914145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[846f1aab-c99b-4ac4-87e2-8b084013369e]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:50.914976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:27:50.915976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:50.916523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:50.916959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" released by "apply_metadata_nat_rules" :: held 0.241s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:27:50.917617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-1764c0b6-02c3-41fb-89a4-7707b146c7b1 Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.haproxy Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 1764c0b6-02c3-41fb-89a4-7707b146c7b1 Jul 01 09:27:50.918206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:27:50.922526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/1764c0b6-02c3-41fb-89a4-7707b146c7b1.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:27:50.964875 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-1764c0b6-02c3-41fb-89a4-7707b146c7b1[127467]: Proxy listener started. Jul 01 09:27:50.972746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 1764c0b6-02c3-41fb-89a4-7707b146c7b1 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:27:50.973170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 1764c0b6-02c3-41fb-89a4-7707b146c7b1, update_id 5dc30a7d-2f3a-46d6-a05e-9fd2b40ab07e. Time elapsed: 2.664 Jul 01 09:27:50.973536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 1764c0b6-02c3-41fb-89a4-7707b146c7b1, action 3, priority 1, update_id 4a5f4df7-0d1a-4264-b709-d4fba9f590d6. Wait time elapsed: 0.912 Jul 01 09:27:50.973901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 1764c0b6-02c3-41fb-89a4-7707b146c7b1, action 3, priority 1, update_id 4a5f4df7-0d1a-4264-b709-d4fba9f590d6. Wait time elapsed: 0.912 Jul 01 09:27:50.974334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:52.116175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.142s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:52.118489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 1764c0b6-02c3-41fb-89a4-7707b146c7b1 info in cache, will do the router update action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:637}} Jul 01 09:27:52.119181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks [] for router (1764c0b6-02c3-41fb-89a4-7707b146c7b1), before_update {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:52.119936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 1764c0b6-02c3-41fb-89a4-7707b146c7b1 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:27:52.174289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62af2378-d11c-4360-9edd-ca4bdbe9b285]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 256600, 'reachable_time': 43560, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127472, 'error': None, 'target': 'qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.177061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:52.177891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:52.178851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:52.179125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:27:52.199237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-08bd7af0-da32-4913-909f-6fbd0713995d tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['8ca431bf-60b6-4cc2-9a4e-c1491a8264c0'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:52.201270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8ca431bf-60b6-4cc2-9a4e-c1491a8264c0, action 3, priority 1, update_id ee784d76-4aed-415b-b7b4-4f782e6b0109. Wait time elapsed: 0.002 Jul 01 09:27:52.201517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8ca431bf-60b6-4cc2-9a4e-c1491a8264c0, action 3, priority 1, update_id ee784d76-4aed-415b-b7b4-4f782e6b0109. Wait time elapsed: 0.002 Jul 01 09:27:52.201835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:52.203314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[886a4ef2-ba76-4ad2-b574-0cfcd6b796a6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 256600, 'reachable_time': 43560, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127473, 'error': None, 'target': 'qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.204739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:52.205205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:27:52.254590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e55a6095-00d5-4135-8945-59bf205347c3]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-INPUT -m mark --mark 0x1/0xffff -j ACCEPT\n-A neutron-l3-agent-INPUT -p tcp -m tcp --dport 9697 -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.310407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f73f6ab-b680-44d4-ab33-27c6be1ccaec]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-INPUT -m mark --mark 0x1/0xffff -j ACCEPT\n-A neutron-l3-agent-INPUT -p tcp -m tcp --dport 9697 -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.313541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 0 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:27:52.313905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:52.314183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:52.315189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" released by "process_external" :: held 0.135s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:27:52.315676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:52.316490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:27:52.317746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:27:52.318542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:27:52.361466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90a4b047-345a-434d-9ed6-33123639e1b0]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-INPUT -m mark --mark 0x1/0xffff -j ACCEPT\n-A neutron-l3-agent-INPUT -p tcp -m tcp --dport 9697 -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.376705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.175s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:52.377135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 8ca431bf-60b6-4cc2-9a4e-c1491a8264c0 was not found. Performing router cleanup Jul 01 09:27:52.379119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d16c2ae5-1edc-4699-825f-e1cbe1af8e85]: (4, ['qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.379958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:52.380590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:52.381001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 8ca431bf-60b6-4cc2-9a4e-c1491a8264c0, update_id ee784d76-4aed-415b-b7b4-4f782e6b0109. Time elapsed: 0.180 Jul 01 09:27:52.408896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d24862f2-e1b8-4d16-831a-3a45a6ad1141]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:27:52 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-INPUT -m mark --mark 0x1/0xffff -j ACCEPT\n-A neutron-l3-agent-INPUT -p tcp -m tcp --dport 9697 -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:27:52 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.412733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 0 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:27:52.413438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:52.424708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:27:52.426106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1" released by "process_address_scope" :: held 0.110s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:27:52.427008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_updated-8767249873071', 'neutron.agent.linux.pd.update_router-8767249872375'] for router (1764c0b6-02c3-41fb-89a4-7707b146c7b1), after_update {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:52.427625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.update_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:52.435663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.update_router" :: held 0.008s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:52.436517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 1764c0b6-02c3-41fb-89a4-7707b146c7b1 update action. {{(pid=106231) update_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:65}} Jul 01 09:27:52.436956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 1764c0b6-02c3-41fb-89a4-7707b146c7b1, update_id 4a5f4df7-0d1a-4264-b709-d4fba9f590d6. Time elapsed: 1.464 Jul 01 09:27:52.612167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-63e76b02-9ad5-42c6-a713-a1fff832dd4e tempest-NetworksTestDHCPv6-1249225972 tempest-NetworksTestDHCPv6-1249225972-project] Got router deleted notification for 1764c0b6-02c3-41fb-89a4-7707b146c7b1 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:27:52.614724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 1764c0b6-02c3-41fb-89a4-7707b146c7b1, action 1, priority 1, update_id efb4f345-6798-4146-b867-1badea3c6cf8. Wait time elapsed: 0.001 Jul 01 09:27:52.614973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 1764c0b6-02c3-41fb-89a4-7707b146c7b1, action 1, priority 1, update_id efb4f345-6798-4146-b867-1badea3c6cf8. Wait time elapsed: 0.001 Jul 01 09:27:52.615276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 1764c0b6-02c3-41fb-89a4-7707b146c7b1 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:27:52.615631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (1764c0b6-02c3-41fb-89a4-7707b146c7b1), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:52.666527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ed97ac4-29b4-4db8-9b03-6b55124b709c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.671525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9b0cd2e-2c63-4660-9694-d3ed09820987]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.676518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9d1b421-3f36-408d-8805-4fab9dd063c9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.676844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 1764c0b6-02c3-41fb-89a4-7707b146c7b1 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:27:52.680132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2214632-19f1-4cdd-b0b8-56bf37bb92af]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.723793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40066835-4d4a-4790-be2e-06bed7d5dd79]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 256600, 'reachable_time': 43560, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127488, 'error': None, 'target': 'qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.725571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:52.725854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:52.772007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7aceb3c-8bbc-4442-a4f1-aad2166e7f01]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 256600, 'reachable_time': 43560, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127498, 'error': None, 'target': 'qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.773965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 1764c0b6-02c3-41fb-89a4-7707b146c7b1 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:27:52.774477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:52.774793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/1764c0b6-02c3-41fb-89a4-7707b146c7b1.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:27:52.775038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 1764c0b6-02c3-41fb-89a4-7707b146c7b1 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:27:52.775797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 1764c0b6-02c3-41fb-89a4-7707b146c7b1 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:27:52.778137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e60daafb-56db-47dc-b26f-a6a56c8f4118]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.802780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4062a10a-89df-444a-8eef-91c0a2b85470]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 256600, 'reachable_time': 43560, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 127504, 'error': None, 'target': 'qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.812520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-1764c0b6-02c3-41fb-89a4-7707b146c7b1 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:27:52.812520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ffc8479-fb9a-477f-a83c-1bca60636bd4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:52.812520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 1764c0b6-02c3-41fb-89a4-7707b146c7b1 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:27:52.812904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (1764c0b6-02c3-41fb-89a4-7707b146c7b1), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:27:52.812904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:52.813154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:52.813453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:52.813717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:52.813965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 1764c0b6-02c3-41fb-89a4-7707b146c7b1, update_id efb4f345-6798-4146-b867-1badea3c6cf8. Time elapsed: 0.199 Jul 01 09:27:55.512169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-6fde4190-29cf-477f-a229-c2b82444a01c tempest-RoutersAdminTest-1423367726 tempest-RoutersAdminTest-1423367726-project] Got routers updated notification :['d95961c7-a977-49e7-a8cd-81ed5941157a'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:55.522432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update d95961c7-a977-49e7-a8cd-81ed5941157a, action 3, priority 1, update_id c9159dd2-b02c-4276-848b-87ec8e335fae. Wait time elapsed: 0.002 Jul 01 09:27:55.523926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for d95961c7-a977-49e7-a8cd-81ed5941157a, action 3, priority 1, update_id c9159dd2-b02c-4276-848b-87ec8e335fae. Wait time elapsed: 0.003 Jul 01 09:27:55.526061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:55.529437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-94f9c6c1-da6e-4e0f-94fb-7245b83b0873 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['8ca431bf-60b6-4cc2-9a4e-c1491a8264c0'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:27:55.532166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8ca431bf-60b6-4cc2-9a4e-c1491a8264c0, action 3, priority 1, update_id 5d8455c3-9590-4dba-a5e4-20f3d94dc154. Wait time elapsed: 0.002 Jul 01 09:27:55.533264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8ca431bf-60b6-4cc2-9a4e-c1491a8264c0, action 3, priority 1, update_id 5d8455c3-9590-4dba-a5e4-20f3d94dc154. Wait time elapsed: 0.003 Jul 01 09:27:55.533794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:27:55.647746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.114s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:55.648218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 8ca431bf-60b6-4cc2-9a4e-c1491a8264c0 was not found. Performing router cleanup Jul 01 09:27:55.649806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60a77927-e90a-4162-8ea1-12d4d8da6b6e]: (4, ['qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:55.650494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:55.650922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:55.651476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 8ca431bf-60b6-4cc2-9a4e-c1491a8264c0, update_id 5d8455c3-9590-4dba-a5e4-20f3d94dc154. Time elapsed: 0.119 Jul 01 09:27:55.908963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.377s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:27:55.910244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router d95961c7-a977-49e7-a8cd-81ed5941157a was not found. Performing router cleanup Jul 01 09:27:55.913265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0cf4ab73-734c-437c-aa28-8c418494bd7c]: (4, ['qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:55.915194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:55.916146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:55.918675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for d95961c7-a977-49e7-a8cd-81ed5941157a, update_id c9159dd2-b02c-4276-848b-87ec8e335fae. Time elapsed: 0.394 Jul 01 09:27:55.931685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b6ca4457-afa3-4955-b1ed-6693cf5c88f5 tempest-RoutersNegativeTest-1826052474 tempest-RoutersNegativeTest-1826052474-project] Got router deleted notification for 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:27:55.932984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, action 1, priority 1, update_id 9bd02b08-fc8c-459e-84f2-e579cc4253cb. Wait time elapsed: 0.001 Jul 01 09:27:55.934014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, action 1, priority 1, update_id 9bd02b08-fc8c-459e-84f2-e579cc4253cb. Wait time elapsed: 0.002 Jul 01 09:27:55.934534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d was not found. Performing router cleanup Jul 01 09:27:55.935890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b62f2eb1-7478-424c-98de-e16da3449a9e]: (4, ['qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:55.944064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:55.944333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:55.944997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 3a6cfb12-2bf3-425a-9848-2dc4a1b4b86d, update_id 9bd02b08-fc8c-459e-84f2-e579cc4253cb. Time elapsed: 0.012 Jul 01 09:27:56.535738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-d3a54c7f-f04a-419f-813b-c844e9df81ea tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got router deleted notification for 8ca431bf-60b6-4cc2-9a4e-c1491a8264c0 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:27:56.536791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8ca431bf-60b6-4cc2-9a4e-c1491a8264c0, action 1, priority 1, update_id dc0bc04a-141c-4880-84a3-5687c847b60a. Wait time elapsed: 0.001 Jul 01 09:27:56.537206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8ca431bf-60b6-4cc2-9a4e-c1491a8264c0, action 1, priority 1, update_id dc0bc04a-141c-4880-84a3-5687c847b60a. Wait time elapsed: 0.001 Jul 01 09:27:56.537466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 8ca431bf-60b6-4cc2-9a4e-c1491a8264c0 was not found. Performing router cleanup Jul 01 09:27:56.544939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3765e9fd-0f0e-40ea-b941-4b69b66ae51f]: (4, ['qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:56.545948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:56.546222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:56.546477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 8ca431bf-60b6-4cc2-9a4e-c1491a8264c0, update_id dc0bc04a-141c-4880-84a3-5687c847b60a. Time elapsed: 0.010 Jul 01 09:27:56.728705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-99e47979-68ef-4911-9102-67a5f0876486 tempest-RoutersAdminTest-1423367726 tempest-RoutersAdminTest-1423367726-project] Got router deleted notification for d95961c7-a977-49e7-a8cd-81ed5941157a {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:27:56.730633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update d95961c7-a977-49e7-a8cd-81ed5941157a, action 1, priority 1, update_id c3e4e457-83f4-4923-bce2-f073116d45e1. Wait time elapsed: 0.001 Jul 01 09:27:56.731080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for d95961c7-a977-49e7-a8cd-81ed5941157a, action 1, priority 1, update_id c3e4e457-83f4-4923-bce2-f073116d45e1. Wait time elapsed: 0.001 Jul 01 09:27:56.732160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router d95961c7-a977-49e7-a8cd-81ed5941157a was not found. Performing router cleanup Jul 01 09:27:56.737625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[020ed72b-5e29-4739-999c-93d5822d45d1]: (4, ['qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:27:56.739116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:27:56.739606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:27:56.740051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for d95961c7-a977-49e7-a8cd-81ed5941157a, update_id c3e4e457-83f4-4923-bce2-f073116d45e1. Time elapsed: 0.009 Jul 01 09:28:05.223953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-580d5e4d-c21f-44b4-bc68-2963951cce9f tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['2f1ea1c8-9040-4664-89f5-16d1d47ea691'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:05.225050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 2f1ea1c8-9040-4664-89f5-16d1d47ea691, action 3, priority 1, update_id ae455850-5acf-41ae-8ca9-51d5d0fca1b1. Wait time elapsed: 0.001 Jul 01 09:28:05.225449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 2f1ea1c8-9040-4664-89f5-16d1d47ea691, action 3, priority 1, update_id ae455850-5acf-41ae-8ca9-51d5d0fca1b1. Wait time elapsed: 0.001 Jul 01 09:28:05.225825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:05.439481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.213s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:05.439985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 2f1ea1c8-9040-4664-89f5-16d1d47ea691 was not found. Performing router cleanup Jul 01 09:28:05.443622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f315a553-28e3-4ea8-96ca-f8c47dd99d3f]: (4, ['qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:05.445119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:05.445971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:05.447572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 2f1ea1c8-9040-4664-89f5-16d1d47ea691, update_id ae455850-5acf-41ae-8ca9-51d5d0fca1b1. Time elapsed: 0.221 Jul 01 09:28:08.004152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-30c478dc-ac91-4c1d-9ac5-94592ba36957 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['2f1ea1c8-9040-4664-89f5-16d1d47ea691'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:08.008654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 2f1ea1c8-9040-4664-89f5-16d1d47ea691, action 3, priority 1, update_id 0475a0b2-5806-4e3c-b1e5-ba92bbc7a175. Wait time elapsed: 0.003 Jul 01 09:28:08.009259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 2f1ea1c8-9040-4664-89f5-16d1d47ea691, action 3, priority 1, update_id 0475a0b2-5806-4e3c-b1e5-ba92bbc7a175. Wait time elapsed: 0.003 Jul 01 09:28:08.010656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:08.230124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.220s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:08.230786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 2f1ea1c8-9040-4664-89f5-16d1d47ea691 was not found. Performing router cleanup Jul 01 09:28:08.233200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a173138-113f-40bd-af1e-b532d387eb2e]: (4, ['qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:08.241214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:08.241753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:08.242241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 2f1ea1c8-9040-4664-89f5-16d1d47ea691, update_id 0475a0b2-5806-4e3c-b1e5-ba92bbc7a175. Time elapsed: 0.234 Jul 01 09:28:08.581759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-6bc114e4-123d-4435-adf2-8bb96913c510 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got router deleted notification for 2f1ea1c8-9040-4664-89f5-16d1d47ea691 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:28:08.582953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 2f1ea1c8-9040-4664-89f5-16d1d47ea691, action 1, priority 1, update_id 0f6c62b2-5c9a-4ee3-ab94-c605e250ecc9. Wait time elapsed: 0.001 Jul 01 09:28:08.583199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 2f1ea1c8-9040-4664-89f5-16d1d47ea691, action 1, priority 1, update_id 0f6c62b2-5c9a-4ee3-ab94-c605e250ecc9. Wait time elapsed: 0.001 Jul 01 09:28:08.583475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 2f1ea1c8-9040-4664-89f5-16d1d47ea691 was not found. Performing router cleanup Jul 01 09:28:08.584953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43cca750-0ac4-4c35-9da5-9b42351329cf]: (4, ['qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:08.588062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:08.588592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:08.589049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 2f1ea1c8-9040-4664-89f5-16d1d47ea691, update_id 0f6c62b2-5c9a-4ee3-ab94-c605e250ecc9. Time elapsed: 0.006 Jul 01 09:28:15.071927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:15.074477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:15.076491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:15.081483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.006s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:15.087813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:28:15.121818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ad7a833c-323e-4128-ac0a-a6da6eceaaa4 tempest-RoutersIpV6AdminTest-896058136 tempest-RoutersIpV6AdminTest-896058136-project-admin] Got routers updated notification :['600bd9be-a076-4b49-ad7c-ef21063194dc'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:15.125036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 600bd9be-a076-4b49-ad7c-ef21063194dc, action 3, priority 1, update_id da8c786f-16cf-43ed-88f6-31ba49dc57d7. Wait time elapsed: 0.001 Jul 01 09:28:15.126126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 600bd9be-a076-4b49-ad7c-ef21063194dc, action 3, priority 1, update_id da8c786f-16cf-43ed-88f6-31ba49dc57d7. Wait time elapsed: 0.001 Jul 01 09:28:15.127142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:15.500773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-3e583368-3b86-4f74-986a-6aecff3db2ee tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['56896fe3-676e-4160-82f2-81f56604ae5a'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:15.502075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 56896fe3-676e-4160-82f2-81f56604ae5a, action 3, priority 1, update_id 0b491ac2-d291-4912-8ee0-80db5e7fb4bf. Wait time elapsed: 0.001 Jul 01 09:28:15.503082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 56896fe3-676e-4160-82f2-81f56604ae5a, action 3, priority 1, update_id 0b491ac2-d291-4912-8ee0-80db5e7fb4bf. Wait time elapsed: 0.001 Jul 01 09:28:15.504046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:15.632422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.129s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:15.633027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 56896fe3-676e-4160-82f2-81f56604ae5a was not found. Performing router cleanup Jul 01 09:28:15.634909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b6d408f-a228-4593-a6fe-8f91b3b44cf9]: (4, ['qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:15.636448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:15.636961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:15.637447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 56896fe3-676e-4160-82f2-81f56604ae5a, update_id 0b491ac2-d291-4912-8ee0-80db5e7fb4bf. Time elapsed: 0.135 Jul 01 09:28:16.651852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c6598bce-30d1-4db9-bb07-8eadd883ab21 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['56896fe3-676e-4160-82f2-81f56604ae5a'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:16.654150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 56896fe3-676e-4160-82f2-81f56604ae5a, action 3, priority 1, update_id 59449d01-8987-446c-8e85-e6f8a9cf8175. Wait time elapsed: 0.001 Jul 01 09:28:16.654632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 56896fe3-676e-4160-82f2-81f56604ae5a, action 3, priority 1, update_id 59449d01-8987-446c-8e85-e6f8a9cf8175. Wait time elapsed: 0.002 Jul 01 09:28:16.655468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:17.376419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.721s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:17.376686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 56896fe3-676e-4160-82f2-81f56604ae5a was not found. Performing router cleanup Jul 01 09:28:17.377929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f34ded41-34c7-4173-a533-2a03eca261c8]: (4, ['qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:17.378661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:17.378780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:17.378942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 56896fe3-676e-4160-82f2-81f56604ae5a, update_id 59449d01-8987-446c-8e85-e6f8a9cf8175. Time elapsed: 0.725 Jul 01 09:28:18.157477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ad34d2b3-c42c-477b-ab97-95886b89d7f2 tempest-RoutersIpV6AdminTest-896058136 tempest-RoutersIpV6AdminTest-896058136-project-admin] Got router deleted notification for 600bd9be-a076-4b49-ad7c-ef21063194dc {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:28:19.010714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ed85e1c2-bbef-4629-ac0f-d64cb3ca21fe tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got router deleted notification for 56896fe3-676e-4160-82f2-81f56604ae5a {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:28:19.012726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 56896fe3-676e-4160-82f2-81f56604ae5a, action 1, priority 1, update_id c347e164-ada1-4b87-ac66-36029e4077a6. Wait time elapsed: 0.001 Jul 01 09:28:19.013262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 56896fe3-676e-4160-82f2-81f56604ae5a, action 1, priority 1, update_id c347e164-ada1-4b87-ac66-36029e4077a6. Wait time elapsed: 0.001 Jul 01 09:28:19.013826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 56896fe3-676e-4160-82f2-81f56604ae5a was not found. Performing router cleanup Jul 01 09:28:19.016042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80f08810-e53d-4dbd-82ba-2f9b1996578c]: (4, ['qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:19.017252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:19.017722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:19.018260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 56896fe3-676e-4160-82f2-81f56604ae5a, update_id c347e164-ada1-4b87-ac66-36029e4077a6. Time elapsed: 0.005 Jul 01 09:28:19.298289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.171s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:19.303425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 600bd9be-a076-4b49-ad7c-ef21063194dc was not found. Performing router cleanup Jul 01 09:28:19.306641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f066a8d0-c394-4937-ba95-f64a3fdd493b]: (4, ['qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:19.308943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:19.312033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:19.312777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 600bd9be-a076-4b49-ad7c-ef21063194dc, update_id da8c786f-16cf-43ed-88f6-31ba49dc57d7. Time elapsed: 4.188 Jul 01 09:28:19.313714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 600bd9be-a076-4b49-ad7c-ef21063194dc, action 1, priority 1, update_id 81ce35db-ed29-42bb-a58b-e959ba504662. Wait time elapsed: 1.155 Jul 01 09:28:19.315015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 600bd9be-a076-4b49-ad7c-ef21063194dc, action 1, priority 1, update_id 81ce35db-ed29-42bb-a58b-e959ba504662. Wait time elapsed: 1.156 Jul 01 09:28:19.316749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 600bd9be-a076-4b49-ad7c-ef21063194dc was not found. Performing router cleanup Jul 01 09:28:19.319694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6297046b-4289-4745-9c5f-cff1e868824e]: (4, ['qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:19.324874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:19.325977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:19.326829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 600bd9be-a076-4b49-ad7c-ef21063194dc, update_id 81ce35db-ed29-42bb-a58b-e959ba504662. Time elapsed: 0.013 Jul 01 09:28:21.700455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-e460a90a-3c4f-45e3-9e38-7db61c9db8e7 tempest-RoutersIpV6AdminTest-896058136 tempest-RoutersIpV6AdminTest-896058136-project-admin] Got router deleted notification for 9a828365-c0d2-4c0a-833c-0d2e3c40cbfa {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:28:21.707147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9a828365-c0d2-4c0a-833c-0d2e3c40cbfa, action 1, priority 1, update_id 635d1a15-a777-449e-aa36-e3f3bc2775c3. Wait time elapsed: 0.004 Jul 01 09:28:21.707643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9a828365-c0d2-4c0a-833c-0d2e3c40cbfa, action 1, priority 1, update_id 635d1a15-a777-449e-aa36-e3f3bc2775c3. Wait time elapsed: 0.005 Jul 01 09:28:21.708266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9a828365-c0d2-4c0a-833c-0d2e3c40cbfa was not found. Performing router cleanup Jul 01 09:28:21.709606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf15986d-a530-4891-b501-634639c8bb54]: (4, ['qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:21.711245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:21.712637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:21.713215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9a828365-c0d2-4c0a-833c-0d2e3c40cbfa, update_id 635d1a15-a777-449e-aa36-e3f3bc2775c3. Time elapsed: 0.007 Jul 01 09:28:22.363487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-bd398229-81eb-4808-8c9f-d5977a2e3315 tempest-TagsExtTest-1485968311 tempest-TagsExtTest-1485968311-project] Got router deleted notification for bf01b19b-7a96-4d58-aac3-1a5003511207 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:28:22.364762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update bf01b19b-7a96-4d58-aac3-1a5003511207, action 1, priority 1, update_id 873831e3-0e28-4e05-817c-81c9c9684d4b. Wait time elapsed: 0.001 Jul 01 09:28:22.366012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for bf01b19b-7a96-4d58-aac3-1a5003511207, action 1, priority 1, update_id 873831e3-0e28-4e05-817c-81c9c9684d4b. Wait time elapsed: 0.001 Jul 01 09:28:22.366519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router bf01b19b-7a96-4d58-aac3-1a5003511207 was not found. Performing router cleanup Jul 01 09:28:22.368106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b28f834e-f28e-429a-b9ec-e117647383fa]: (4, ['qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:22.372548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:22.373118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:22.373543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for bf01b19b-7a96-4d58-aac3-1a5003511207, update_id 873831e3-0e28-4e05-817c-81c9c9684d4b. Time elapsed: 0.009 Jul 01 09:28:26.153003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-f3e65b67-5ae6-4e12-97de-01aa63039ab5 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['e4d896ce-9b16-4da0-8118-55464b82b4bd'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:26.159269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4d896ce-9b16-4da0-8118-55464b82b4bd, action 3, priority 1, update_id 6faebf1f-cdaf-4f1d-a8c5-36bcde0aebbf. Wait time elapsed: 0.005 Jul 01 09:28:26.161384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4d896ce-9b16-4da0-8118-55464b82b4bd, action 3, priority 1, update_id 6faebf1f-cdaf-4f1d-a8c5-36bcde0aebbf. Wait time elapsed: 0.005 Jul 01 09:28:26.162108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:26.297174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.134s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:26.297174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router e4d896ce-9b16-4da0-8118-55464b82b4bd was not found. Performing router cleanup Jul 01 09:28:26.299179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c0ce80c-3545-46e9-a709-bb602d4f57d0]: (4, ['qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:26.300669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:26.300941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:26.301205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for e4d896ce-9b16-4da0-8118-55464b82b4bd, update_id 6faebf1f-cdaf-4f1d-a8c5-36bcde0aebbf. Time elapsed: 0.142 Jul 01 09:28:27.569332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-727cab05-7c22-4d36-99c5-254ca41dd239 tempest-RoutersIpV6AdminTest-1507210859 tempest-RoutersIpV6AdminTest-1507210859-project] Got routers updated notification :['da5aa758-a757-4c0b-ab57-4f6f41e767b1'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:27.571219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update da5aa758-a757-4c0b-ab57-4f6f41e767b1, action 3, priority 1, update_id 091754a1-15d8-4353-b0d7-9b15941d3ed4. Wait time elapsed: 0.001 Jul 01 09:28:27.571765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for da5aa758-a757-4c0b-ab57-4f6f41e767b1, action 3, priority 1, update_id 091754a1-15d8-4353-b0d7-9b15941d3ed4. Wait time elapsed: 0.001 Jul 01 09:28:27.572284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:27.801064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.229s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:27.801426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router da5aa758-a757-4c0b-ab57-4f6f41e767b1 was not found. Performing router cleanup Jul 01 09:28:27.802626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ab9ee9a-e357-4227-ae96-71d89b561651]: (4, ['qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:27.803210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:27.803771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:27.804137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for da5aa758-a757-4c0b-ab57-4f6f41e767b1, update_id 091754a1-15d8-4353-b0d7-9b15941d3ed4. Time elapsed: 0.233 Jul 01 09:28:29.775651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-903daa93-c182-4b6b-8f0c-3cb5ed455a63 tempest-RoutersIpV6AdminTest-1507210859 tempest-RoutersIpV6AdminTest-1507210859-project] Got router deleted notification for da5aa758-a757-4c0b-ab57-4f6f41e767b1 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:28:29.778671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update da5aa758-a757-4c0b-ab57-4f6f41e767b1, action 1, priority 1, update_id 11b99843-cf3d-489c-bfd9-f2e40d590b35. Wait time elapsed: 0.001 Jul 01 09:28:29.779090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for da5aa758-a757-4c0b-ab57-4f6f41e767b1, action 1, priority 1, update_id 11b99843-cf3d-489c-bfd9-f2e40d590b35. Wait time elapsed: 0.001 Jul 01 09:28:29.779705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router da5aa758-a757-4c0b-ab57-4f6f41e767b1 was not found. Performing router cleanup Jul 01 09:28:29.781415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5200cae-dac8-4ca0-ac56-f824a5d88500]: (4, ['qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:29.782441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:29.782864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:29.783277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for da5aa758-a757-4c0b-ab57-4f6f41e767b1, update_id 11b99843-cf3d-489c-bfd9-f2e40d590b35. Time elapsed: 0.005 Jul 01 09:28:30.911202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-d855f5a7-0a96-4b98-ab96-05c288d0ec6b tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['e4d896ce-9b16-4da0-8118-55464b82b4bd'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:30.912913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4d896ce-9b16-4da0-8118-55464b82b4bd, action 3, priority 1, update_id 8c021fa0-ec44-4c46-bfb7-cd8a4a3bd90f. Wait time elapsed: 0.001 Jul 01 09:28:30.913361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4d896ce-9b16-4da0-8118-55464b82b4bd, action 3, priority 1, update_id 8c021fa0-ec44-4c46-bfb7-cd8a4a3bd90f. Wait time elapsed: 0.001 Jul 01 09:28:30.915151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:31.196502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.282s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:31.197140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router e4d896ce-9b16-4da0-8118-55464b82b4bd was not found. Performing router cleanup Jul 01 09:28:31.198562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b9b7fba-49f0-46f2-a2fc-e314488ac046]: (4, ['qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:31.199047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:31.199313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:31.199912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for e4d896ce-9b16-4da0-8118-55464b82b4bd, update_id 8c021fa0-ec44-4c46-bfb7-cd8a4a3bd90f. Time elapsed: 0.287 Jul 01 09:28:31.693129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-eaaa489a-e44d-4706-80ee-cdf043bbc131 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got router deleted notification for e4d896ce-9b16-4da0-8118-55464b82b4bd {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:28:31.698004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4d896ce-9b16-4da0-8118-55464b82b4bd, action 1, priority 1, update_id 7dff1561-95d8-4d5b-9b1f-7358bf402b1a. Wait time elapsed: 0.004 Jul 01 09:28:31.707017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4d896ce-9b16-4da0-8118-55464b82b4bd, action 1, priority 1, update_id 7dff1561-95d8-4d5b-9b1f-7358bf402b1a. Wait time elapsed: 0.013 Jul 01 09:28:31.707537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router e4d896ce-9b16-4da0-8118-55464b82b4bd was not found. Performing router cleanup Jul 01 09:28:31.709082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ee95df2-a7b5-4800-b77e-9ed6eb257d4d]: (4, ['qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:31.709881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:31.710342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:31.710949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for e4d896ce-9b16-4da0-8118-55464b82b4bd, update_id 7dff1561-95d8-4d5b-9b1f-7358bf402b1a. Time elapsed: 0.013 Jul 01 09:28:34.200959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-7bcfca19-fb31-4ecb-90c0-e314cb574fd4 tempest-RoutersIpV6AdminTest-896058136 tempest-RoutersIpV6AdminTest-896058136-project-admin] Got routers updated notification :['ac3ad198-c5b8-4acf-afc0-b5881b8bb080'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:34.203280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update ac3ad198-c5b8-4acf-afc0-b5881b8bb080, action 3, priority 1, update_id 186505c5-23ef-46f9-ad69-34dbdf92432f. Wait time elapsed: 0.001 Jul 01 09:28:34.203755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for ac3ad198-c5b8-4acf-afc0-b5881b8bb080, action 3, priority 1, update_id 186505c5-23ef-46f9-ad69-34dbdf92432f. Wait time elapsed: 0.001 Jul 01 09:28:34.204697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:37.257184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.046s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:37.258311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router ac3ad198-c5b8-4acf-afc0-b5881b8bb080 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:28:37.259494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (ac3ad198-c5b8-4acf-afc0-b5881b8bb080), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:37.260497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:37.260959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:37.262345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60376661-2dee-48e9-8552-8ec0561f255d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.306042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3c33d5d-78a7-4e9a-aca0-805a172f4b1e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.342392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30290c55-ca28-4980-b41c-bc89a5361dd9]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.387726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6bdd9734-6bbd-40b6-9c70-841d4216583a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128080, 'error': None, 'target': 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.429186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdfbf698-35b4-438f-be79-0f3345381300]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.453771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a5514ff-6dfd-45a5-bb27-945f5a3dfaaf]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.491869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[257071a3-82fa-489b-8b8c-10fb072e718a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.518100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e555ee72-f24f-4dba-a150-dbde307526f5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.519216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:37.519854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:37.520294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router ac3ad198-c5b8-4acf-afc0-b5881b8bb080 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:28:37.558593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6685047d-de28-4ca5-808a-28f4f50aee57]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261364, 'reachable_time': 28932, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128086, 'error': None, 'target': 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.560113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:37.561070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:37.561680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:37.562269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:28:37.562594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:37.562856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:37.563297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'c7fd2ece-94a5-42ed-a371-a26a02da29e2', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:42:75:96', 'admin_state_up': True, 'status': 'DOWN', 'device_id': 'ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'device_owner': 'network:router_gateway', 'standard_attr_id': 958, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.21', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:28:30Z', 'updated_at': '2022-07-01T09:28:35Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-c7fd2ece-94), ns(qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:28:37.586934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:28:37.587251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23e52b0a-bd22-42d4-850f-86a3fccebf61]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.594427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ebce692-202a-4758-bf24-9f3a8c6b15ae]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.595456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-c7fd2ece-94, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:37.596342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:28:37.597205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-c7fd2ece-94, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:37.597863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-c7fd2ece-94, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:37.598257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-c7fd2ece-94, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:37.599017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-c7fd2ece-94, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:37.599418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-c7fd2ece-94, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'c7fd2ece-94a5-42ed-a371-a26a02da29e2', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:75:96'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:37.661598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f3f1fef-f97d-45d2-addc-7e8c5017389c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.712126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[48a59af7-0356-42b1-acab-7e97558490f7]: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:28:37.712126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:28:37.712126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:28:37.712126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:28:37.712126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:28:37.712126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:28:37.712126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:28:37.712126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:28:37.712126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:28:37.712126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:28:37.712126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:28:37.712126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:28:37.712126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:37.714475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48a59af7-0356-42b1-acab-7e97558490f7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.715203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:37.716036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:28:37.747067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-e8742f67-f82d-469a-8728-1cd36c35e9e9 tempest-RoutersIpV6AdminTest-896058136 tempest-RoutersIpV6AdminTest-896058136-project-admin] Got routers updated notification :['6093ab8c-22c0-4922-942d-31a6e4c830b9'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:37.748670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 6093ab8c-22c0-4922-942d-31a6e4c830b9, action 3, priority 1, update_id c91b8347-e6c0-47f9-9dee-ec8da90bf5c9. Wait time elapsed: 0.001 Jul 01 09:28:37.749033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 6093ab8c-22c0-4922-942d-31a6e4c830b9, action 3, priority 1, update_id c91b8347-e6c0-47f9-9dee-ec8da90bf5c9. Wait time elapsed: 0.002 Jul 01 09:28:37.749415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:37.753552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b1f47f6-bd75-414a-9452-872344759870]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.957043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1074ccee-ff05-4b3a-9c08-df5ce5f071be]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:37.994182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d26ca95-c636-4c16-ad63-77af1342e9df]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:38.102225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b5ba3059-3a8b-4a81-a303-e18f2279261f tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['91f9f498-d760-4ea7-84fe-31c1098b6c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:38.103476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id acc46d94-5045-442a-934d-1c70f6ede10b. Wait time elapsed: 0.001 Jul 01 09:28:38.103886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id acc46d94-5045-442a-934d-1c70f6ede10b. Wait time elapsed: 0.001 Jul 01 09:28:38.104358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:38.210344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f74d61b4-a9d7-435b-bf53-8bd954772c53]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:38.212659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:28:38.214112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:38.214823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:38.215279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" released by "process_external" :: held 0.653s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:38.216304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router ac3ad198-c5b8-4acf-afc0-b5881b8bb080: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:38.224405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:28:38.231924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a0b5877-bf9d-42ef-ac1e-75468bd41ef7]: (4, ['qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:38.233357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:38.234254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:38.234666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for ac3ad198-c5b8-4acf-afc0-b5881b8bb080 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:28:38.235243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:38.242497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d09b5b06-b9ea-43a4-82c6-277d0b061179]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:38.247015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05fe1bb7-0ed1-4e8b-b89d-4d6d23358281]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:38.248266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ef8306a-8b97-4e5b-af8b-bc9c419bb37b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:38.279225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06b3e205-1b61-474a-9917-de2480689707]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261364, 'reachable_time': 28932, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128109, 'error': None, 'target': 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:38.285405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:28:38.285718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d3f0102-49eb-47a4-a99d-52d02f862830]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: ac3ad198-c5b8-4acf-afc0-b5881b8bb080: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:38.286958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ac3ad198-c5b8-4acf-afc0-b5881b8bb080", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-snat-router-1888983133", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "5f686141fcf345f0b7941afe340c0797", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.21" Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": false Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c7fd2ece-94a5-42ed-a371-a26a02da29e2", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:29Z", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:32Z", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "5f686141fcf345f0b7941afe340c0797", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c7fd2ece-94a5-42ed-a371-a26a02da29e2", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:75:96", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "ac3ad198-c5b8-4acf-afc0-b5881b8bb080", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 958, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.21", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:30Z", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:35Z", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": false, Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update ac3ad198-c5b8-4acf-afc0-b5881b8bb080, action 3, priority 2, update_id 186505c5-23ef-46f9-ad69-34dbdf92432f. Wait time elapsed: 4.088 Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for ac3ad198-c5b8-4acf-afc0-b5881b8bb080, action 3, priority 2, update_id 186505c5-23ef-46f9-ad69-34dbdf92432f. Wait time elapsed: 4.089 Jul 01 09:28:38.292422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:40.165008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.415s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:40.165988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 6093ab8c-22c0-4922-942d-31a6e4c830b9 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:28:40.166965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (6093ab8c-22c0-4922-942d-31a6e4c830b9), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:40.167838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:40.168180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:40.170426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.879s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:40.170693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router ac3ad198-c5b8-4acf-afc0-b5881b8bb080 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:28:40.171589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (ac3ad198-c5b8-4acf-afc0-b5881b8bb080), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:40.171895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:40.172166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:40.172678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3088baec-27f2-471f-a66e-0ab8a2bc39ff]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.172995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[332bc13b-b347-4ffe-b883-ff066c478244]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.191665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c05b9d2-3896-484f-8cc4-a2df4fc6a4e7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.200378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38573491-30cb-4b0b-b4f6-324233f60647]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.229930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff3dc9fa-3af1-47c9-a501-4a6f0722b2fb]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.244104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a25d495-bb37-42e3-912e-dce60b0e937a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.290874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9848b2e-8662-4b56-a3f8-49c7cbfa7e63]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128178, 'error': None, 'target': 'qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.317518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[906ce420-e5e1-4e84-948f-a59627192787]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128180, 'error': None, 'target': 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.343932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0be7ab68-d126-42ed-abbc-ea483bb4f8c4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.353482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5209febd-774e-4aad-a1e5-b8f7ccc4a1a1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.383464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d94c76b6-8d67-4f6c-9786-370a15fdacf3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.391181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a79110d5-555f-4a37-b20e-24524a34e420]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.422434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[442bbeb1-8d51-4399-879d-317c3866dfaf]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.434795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c31e8357-62c9-475e-a419-90f55df310c4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.473762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29aabf42-69fe-4ae0-8d34-bd47c4b256f6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.474487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32990f32-a88d-4cac-abd0-dc152dc399a3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.475138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:40.475429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:40.475853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router ac3ad198-c5b8-4acf-afc0-b5881b8bb080 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:28:40.476678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:40.477103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:40.477575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 6093ab8c-22c0-4922-942d-31a6e4c830b9 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:28:40.509890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05cd6e4b-6ce3-4b27-83b0-ae74e5c0cf3a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261658, 'reachable_time': 35848, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128190, 'error': None, 'target': 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.511030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:40.511501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:40.512090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:40.512508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:28:40.513012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:40.513439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:40.513883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'c7fd2ece-94a5-42ed-a371-a26a02da29e2', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:42:75:96', 'admin_state_up': True, 'status': 'DOWN', 'device_id': 'ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'device_owner': 'network:router_gateway', 'standard_attr_id': 958, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.21', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:28:30Z', 'updated_at': '2022-07-01T09:28:35Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-c7fd2ece-94), ns(qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:28:40.534119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[060bba67-8c73-49dd-b215-4ae3491454b5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261654, 'reachable_time': 26076, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128191, 'error': None, 'target': 'qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.534887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:40.535163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:40.535614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:40.535875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:28:40.554579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:28:40.556647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5202005e-1a64-43cc-89a9-c6592bf1a1b0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.563786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26f2bce8-1178-40aa-9b37-fb00e4e5418d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.567993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-c7fd2ece-94, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:40.584304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5dea332-4fba-47c1-b9c0-3f808460ba6d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261654, 'reachable_time': 26076, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128193, 'error': None, 'target': 'qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.585599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:40.588635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:28:40.599407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-62f2ec89-08d9-4d9a-a11c-b7cb3aa02664 tempest-RoutersIpV6AdminTest-896058136 tempest-RoutersIpV6AdminTest-896058136-project-admin] Got router deleted notification for 6093ab8c-22c0-4922-942d-31a6e4c830b9 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:28:40.604972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-c7fd2ece-94, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:40.606671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-c7fd2ece-94, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:40.607311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-c7fd2ece-94, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:40.610100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-c7fd2ece-94, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:40.611960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-c7fd2ece-94, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'c7fd2ece-94a5-42ed-a371-a26a02da29e2', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:75:96'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:40.649183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[266d2773-ca0d-45c9-8093-f1cf78c4bc3a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.663616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69562b66-d15e-4620-852f-5c824c214fa9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.713005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4bb6f42a-c97a-45e2-9d6c-238dad334919]: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:28:40.713005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:28:40.713005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:28:40.713005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:28:40.713005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:28:40.713005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:28:40.713005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:28:40.713005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:28:40.713005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:28:40.713005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:28:40.713005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:28:40.713005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:28:40.713005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:40.715160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4bb6f42a-c97a-45e2-9d6c-238dad334919]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.715903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:40.716509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:28:40.762968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73942274-4b6d-44cf-85cc-e518dc4c0fcf]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.803655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82b8dddd-9adc-4ffc-abce-56c4fd38470b]: (4, ('', 'Another app is currently holding the xtables lock. Perhaps you want to use the -w option?\n', 4)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.854446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8cd6ff2-f7ff-4960-96c4-fa9245b212ba]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:40.884378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e832ff06-7f5c-495f-bac1-1ba34b756e3f]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.100719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8102c05a-4c6a-4d4d-a90d-24f72baa629d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.101791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:28:41.102578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:41.103872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:41.106649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" released by "process_external" :: held 0.569s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:28:41.107288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:41.107725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:28:41.108346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:41.109863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:28:41.165033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46bfd8bc-1c99-4974-b004-f940f0501f67]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.230176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab199ea3-3daf-4e4f-9dbe-b681e1363e74]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.251086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[360147cf-ac43-411b-bf72-b8a283d889ff]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.312110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6abafb2e-f28b-4777-8f14-525fcfba8790]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.312614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:28:41.312929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:41.313196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:41.313485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" released by "process_address_scope" :: held 0.206s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:28:41.313962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (6093ab8c-22c0-4922-942d-31a6e4c830b9), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:41.314589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:41.315080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:28:41.315630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:41.316119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:28:41.352424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5354a454-85a4-465d-89eb-b8af689e90cd]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.424466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80867d4d-d646-46f9-b824-1cafbe411a16]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.462909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[119407de-329f-40f8-bdf4-9c2df4ceee4c]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:41 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:28:41 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.509322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23646f88-e368-459a-bef1-fc7bee55093a]: (4, ('', 'Another app is currently holding the xtables lock. Perhaps you want to use the -w option?\n', 4)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.594809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c9cfac3-679c-400f-bbb1-459ee55c1003]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.655465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2de485dc-3f76-4166-a369-5cc42f91f7fd]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.877503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfd899d4-72b1-48dc-8a4f-6b21278ef7df]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.878233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:28:41.878682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:41.879171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:41.879487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" released by "process_external" :: held 1.367s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:41.884116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router ac3ad198-c5b8-4acf-afc0-b5881b8bb080: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:28:41.887117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffccb0e7-8545-42ff-b508-23248b2d96e5]: (4, ['qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.890711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:41.890711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:41.890711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for ac3ad198-c5b8-4acf-afc0-b5881b8bb080 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:28:41.890711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:41.890711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d57fc268-4b56-44b4-8c19-127dcd06a977]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.893126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1385a9e7-1471-4e5a-8c83-8b498f859fe0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.894808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a64d82a-6afa-4c7e-ac19-dac23b6572d7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.922698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fedf46e-f072-4679-acf2-98118ff33c23]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261658, 'reachable_time': 35848, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128224, 'error': None, 'target': 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.928241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:28:41.928558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[940753ed-3353-4cfb-804f-8c6530b5118a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: ac3ad198-c5b8-4acf-afc0-b5881b8bb080: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-c7fd2ece-94 not found in namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080. Jul 01 09:28:41.929803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ac3ad198-c5b8-4acf-afc0-b5881b8bb080", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-snat-router-1888983133", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "5f686141fcf345f0b7941afe340c0797", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.21" Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": false Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c7fd2ece-94a5-42ed-a371-a26a02da29e2", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:29Z", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:32Z", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "5f686141fcf345f0b7941afe340c0797", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c7fd2ece-94a5-42ed-a371-a26a02da29e2", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:75:96", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "ac3ad198-c5b8-4acf-afc0-b5881b8bb080", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 958, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.21", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:30Z", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:35Z", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": false, Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update ac3ad198-c5b8-4acf-afc0-b5881b8bb080, action 3, priority 2, update_id 186505c5-23ef-46f9-ad69-34dbdf92432f. Wait time elapsed: 7.729 Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for ac3ad198-c5b8-4acf-afc0-b5881b8bb080, action 3, priority 2, update_id 186505c5-23ef-46f9-ad69-34dbdf92432f. Wait time elapsed: 7.730 Jul 01 09:28:41.933802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:41.974562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72dfa53e-a54d-4819-b899-cf73d102dd6a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:41.975463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:28:41.979669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:41.980050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:41.980445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9" released by "apply_metadata_nat_rules" :: held 0.665s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:28:41.981112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/6093ab8c-22c0-4922-942d-31a6e4c830b9.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/6093ab8c-22c0-4922-942d-31a6e4c830b9.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-6093ab8c-22c0-4922-942d-31a6e4c830b9 Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/6093ab8c-22c0-4922-942d-31a6e4c830b9.pid.haproxy Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 6093ab8c-22c0-4922-942d-31a6e4c830b9 Jul 01 09:28:41.981658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:28:41.984246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/6093ab8c-22c0-4922-942d-31a6e4c830b9.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:28:42.039126 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-6093ab8c-22c0-4922-942d-31a6e4c830b9[128226]: Proxy listener started. Jul 01 09:28:42.052764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 6093ab8c-22c0-4922-942d-31a6e4c830b9 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:28:42.053150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 6093ab8c-22c0-4922-942d-31a6e4c830b9, update_id c91b8347-e6c0-47f9-9dee-ec8da90bf5c9. Time elapsed: 4.305 Jul 01 09:28:42.053483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 6093ab8c-22c0-4922-942d-31a6e4c830b9, action 1, priority 1, update_id 2509b893-e9e4-4d41-af63-0844ba45fbb9. Wait time elapsed: 1.453 Jul 01 09:28:42.053821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 6093ab8c-22c0-4922-942d-31a6e4c830b9, action 1, priority 1, update_id 2509b893-e9e4-4d41-af63-0844ba45fbb9. Wait time elapsed: 1.454 Jul 01 09:28:42.054182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 6093ab8c-22c0-4922-942d-31a6e4c830b9 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:28:42.055101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (6093ab8c-22c0-4922-942d-31a6e4c830b9), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:42.097476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[641eb732-330f-4128-828f-fc0de1967e21]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:42.101532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b65aeb65-6f36-480b-90ad-ae23960b65ae]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:42.103659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c6e7dd8-5797-4bf1-81d6-2aa80f5770c6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:42.104655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 6093ab8c-22c0-4922-942d-31a6e4c830b9 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:28:42.105653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67a29bd0-94ec-4d86-aee7-65bd8aefec19]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:42.150309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7ff2c81-4da5-4dfb-b830-77358ff2248b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261654, 'reachable_time': 26076, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128232, 'error': None, 'target': 'qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:42.152037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:42.152521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:42.182650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5b60a4b-6d5e-484b-96c4-ff91b32eba95]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261654, 'reachable_time': 26076, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128233, 'error': None, 'target': 'qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:42.184604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 6093ab8c-22c0-4922-942d-31a6e4c830b9 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:28:42.185528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/6093ab8c-22c0-4922-942d-31a6e4c830b9.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/6093ab8c-22c0-4922-942d-31a6e4c830b9.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:42.186061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/6093ab8c-22c0-4922-942d-31a6e4c830b9.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/6093ab8c-22c0-4922-942d-31a6e4c830b9.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:42.187215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 6093ab8c-22c0-4922-942d-31a6e4c830b9 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:28:42.188225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 6093ab8c-22c0-4922-942d-31a6e4c830b9 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:28:42.189910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b571799-b2ab-4207-ba68-eafc98fbca1c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:42.214766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b894c69-375d-4c3e-a5c0-816476f38f1e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261654, 'reachable_time': 26076, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128234, 'error': None, 'target': 'qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:42.220263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-6093ab8c-22c0-4922-942d-31a6e4c830b9 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:28:42.221438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e5bd37e-7c99-476d-a1c6-8d2854dbe036]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:42.221990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 6093ab8c-22c0-4922-942d-31a6e4c830b9 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:28:42.223138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (6093ab8c-22c0-4922-942d-31a6e4c830b9), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:42.223138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:42.223412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:42.223637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:42.224224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:42.224224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 6093ab8c-22c0-4922-942d-31a6e4c830b9, update_id 2509b893-e9e4-4d41-af63-0844ba45fbb9. Time elapsed: 0.171 Jul 01 09:28:42.917537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-721d2159-dec6-4ac1-a1c1-fd23a9f2cf8c tempest-RoutersIpV6AdminTest-896058136 tempest-RoutersIpV6AdminTest-896058136-project-admin] Got router deleted notification for ac3ad198-c5b8-4acf-afc0-b5881b8bb080 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:28:43.185274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.251s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:43.186578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router ac3ad198-c5b8-4acf-afc0-b5881b8bb080 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:28:43.190004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (ac3ad198-c5b8-4acf-afc0-b5881b8bb080), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:43.191893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:43.192753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:43.199466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[731134d8-9112-468f-b0ff-f0af4299849b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.235786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8169decc-9847-4b5f-ba69-d7cc65fa4efb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.278199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57617b8c-bcca-4d9e-8504-beab013179cb]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.335824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f75685f-a10f-4520-a7e0-0553298c71c5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128238, 'error': None, 'target': 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.383724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0350d5cb-dde3-4b97-aa95-3e9f13bd3612]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.404950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.300s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:43.410541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 91f9f498-d760-4ea7-84fe-31c1098b6c91 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:28:43.414578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:43.415343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:43.416064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:43.417005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[181f2e26-d97a-4378-9766-ae880f2de0df]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.417553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf6a7361-9ec6-422f-9732-2c612cddea6f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.448449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6592c34-36f1-4cd1-bb2c-a3b5f8d5c03f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.462797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e76c4ad-afb5-4c44-b497-daf62f9d05ff]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.475172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c21b188-a62b-43c0-8028-54adb33cc74e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.491301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6be04229-3802-404b-be42-db3fdb2ab290]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.493533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:43.494055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:43.494690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router ac3ad198-c5b8-4acf-afc0-b5881b8bb080 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:28:43.519717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6555a063-c15a-4533-befc-4aba518ca0e0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261959, 'reachable_time': 15672, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128249, 'error': None, 'target': 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.520526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:43.520900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:43.521339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:43.521602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:28:43.534155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a4ed5ac-7954-4aa0-b7d5-90a44287be15]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128247, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.552134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9caa364-3b6f-47ff-a4f5-b9d83bd034a7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261959, 'reachable_time': 15672, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128250, 'error': None, 'target': 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.552968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:43.553473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:28:43.579452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07b604a5-068f-42ae-951c-4f8c636937de]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.600367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[199000a5-9efa-4682-ba41-434d21b8c2ec]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.609592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03eb7cb2-5fea-4622-8bf1-45dec9b1591b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.640558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5729e4b-b0da-457b-9174-225aa74a17b1]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.668960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a70b7642-8333-49ab-b315-26a9bf7df3a2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.669868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:43.670327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:43.670774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:28:43.671710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:28:43.708331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:28:43.709673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a080d54f-03e3-4105-a62d-0de06b72b810]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.714467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0a2f11d-6c43-4dce-b551-6d8bc877c917]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.718253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1b4d4e48-4f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:43.720126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:28:43.722170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1b4d4e48-4f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:43.723117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:43.724662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:43.725358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:43.728358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1b4d4e48-4f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:82:bc:fd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:43.788952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8af7216d-c105-4796-aee4-dc8d278accbd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.828657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8d2ab6b3-6652-484e-9a1f-cff62aa76e5f]: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:28:43.828657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:28:43.828657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:28:43.828657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:28:43.828657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:28:43.828657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:28:43.828657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:28:43.828657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:28:43.828657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:28:43.828657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:28:43.828657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:28:43.828657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:28:43.828657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:43.829707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d2ab6b3-6652-484e-9a1f-cff62aa76e5f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:43.831056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:43.838588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:28:43.847302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d73c5d47-bc6d-45ef-854f-06e01cd234c8]: (4, ['qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.847302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1083f4e2-50fb-40dd-9aa5-009d9fc9a84d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.847302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:43.847302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:43.847302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:28:43.847302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:43.847302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d21273d-a824-4f2c-964a-b5ce658221e0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.850076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f677c6fa-eb59-4343-945d-c67ea48062a0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.851424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e864b99e-8375-430a-85db-05684dc7a70b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.875673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72269b42-9736-4e05-be3a-e8f875d6e956]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.895732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ef69310-d491-4a60-a136-00bfdd8e8ab5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261979, 'reachable_time': 16164, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128272, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.899820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:28:43.899820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a66c67f-bfb1-4038-b666-3244684a5a6a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:43.900928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersIpV6Test-router-1953029727", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:33Z", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:36Z", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 2, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a4a83b6a-52b0-4440-b682-75e82dd825c2", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:bc:fd", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 969, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8::1", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:35Z", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:41Z", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8::/64", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8::1", Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:28:43.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 2, update_id acc46d94-5045-442a-934d-1c70f6ede10b. Wait time elapsed: 5.802 Jul 01 09:28:43.915752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 2, update_id acc46d94-5045-442a-934d-1c70f6ede10b. Wait time elapsed: 5.802 Jul 01 09:28:43.915752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:44.062750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-0cda1a99-cd6c-4147-bb7d-055fc0cfb2eb tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['91f9f498-d760-4ea7-84fe-31c1098b6c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:44.133694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e726f9d4-d925-4378-8a2c-7036474b6c66]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.134263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:28:44.135861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:44.136199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:44.136501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" released by "process_external" :: held 0.615s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:28:44.137010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:44.137276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:28:44.137709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:44.138221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:28:44.182179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3ccd054-7c28-41a5-bf40-11ff7a70d70b]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.230006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ac9feaa-e36f-407a-8bab-161634405bb7]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.260675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc469d18-88d6-4537-9cff-d8e05b1bcf08]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.315327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c10c212f-6a86-435a-b77e-e61a1cae5bfa]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.316339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:28:44.316870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:44.317514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:44.318628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" released by "process_address_scope" :: held 0.181s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:28:44.319206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (ac3ad198-c5b8-4acf-afc0-b5881b8bb080), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:44.320110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:44.320560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:28:44.321158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:28:44.321801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:28:44.379616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5597596-ed79-4590-a44c-8ee1ee1c2878]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.458191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f1107ac-0734-402c-bbb4-ae89efe2ef39]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.499854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32c8727d-948a-4f84-8309-1cd733ed8fd6]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:28:44 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:28:44 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.557833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63f228eb-d90c-40b1-9847-bf4461bf2245]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.558864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:28:44.559421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:44.559845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:28:44.560296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080" released by "apply_metadata_nat_rules" :: held 0.240s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:28:44.561000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-ac3ad198-c5b8-4acf-afc0-b5881b8bb080 Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.haproxy Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID ac3ad198-c5b8-4acf-afc0-b5881b8bb080 Jul 01 09:28:44.561917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:28:44.565262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:28:44.619247 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-ac3ad198-c5b8-4acf-afc0-b5881b8bb080[128295]: Proxy listener started. Jul 01 09:28:44.636245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router ac3ad198-c5b8-4acf-afc0-b5881b8bb080 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:28:44.636795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for ac3ad198-c5b8-4acf-afc0-b5881b8bb080, update_id 186505c5-23ef-46f9-ad69-34dbdf92432f. Time elapsed: 2.705 Jul 01 09:28:44.637256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update ac3ad198-c5b8-4acf-afc0-b5881b8bb080, action 1, priority 1, update_id fd75b2d7-6d16-44c5-b7bf-290a4906f238. Wait time elapsed: 1.719 Jul 01 09:28:44.637661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for ac3ad198-c5b8-4acf-afc0-b5881b8bb080, action 1, priority 1, update_id fd75b2d7-6d16-44c5-b7bf-290a4906f238. Wait time elapsed: 1.719 Jul 01 09:28:44.638271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router ac3ad198-c5b8-4acf-afc0-b5881b8bb080 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:28:44.638742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (ac3ad198-c5b8-4acf-afc0-b5881b8bb080), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:44.675949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0518f499-1ef0-4e11-a8f7-c50d8c01ebe2]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.677711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9b2b009-ad7a-4026-ba53-df6f8659535b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.678807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97cfbc18-5409-4525-9a98-949b676ceac1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.679236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router ac3ad198-c5b8-4acf-afc0-b5881b8bb080 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:28:44.680161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b13d394-2fb8-41f5-b26f-232d9120168b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.716247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f02244e-068c-43d1-af69-c05d3c3da4cc]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261959, 'reachable_time': 15672, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128301, 'error': None, 'target': 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.717019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:44.717301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:44.744393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19ff275b-86a2-4c97-b9bd-01eedf4f1892]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261959, 'reachable_time': 15672, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128302, 'error': None, 'target': 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.745555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: ac3ad198-c5b8-4acf-afc0-b5881b8bb080 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:28:44.746266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:44.748162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/ac3ad198-c5b8-4acf-afc0-b5881b8bb080.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:44.748634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for ac3ad198-c5b8-4acf-afc0-b5881b8bb080 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:28:44.749589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router ac3ad198-c5b8-4acf-afc0-b5881b8bb080 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:28:44.750961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4373adfa-dd87-4baf-84c7-5486896fa6f9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.780339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f11b176c-b927-473c-9f25-bf10edb75235]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 261959, 'reachable_time': 15672, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128305, 'error': None, 'target': 'qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.790939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-ac3ad198-c5b8-4acf-afc0-b5881b8bb080 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:28:44.791110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eab5f5ca-f8ae-4225-afca-ff33688772eb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:44.791611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info ac3ad198-c5b8-4acf-afc0-b5881b8bb080 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:28:44.792174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (ac3ad198-c5b8-4acf-afc0-b5881b8bb080), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:44.793146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:44.793146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:44.793899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:44.794349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:44.795937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for ac3ad198-c5b8-4acf-afc0-b5881b8bb080, update_id fd75b2d7-6d16-44c5-b7bf-290a4906f238. Time elapsed: 0.159 Jul 01 09:28:45.925482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.020s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:45.926459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 91f9f498-d760-4ea7-84fe-31c1098b6c91 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:28:45.927582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:45.928094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:45.928477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:45.931744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d268e347-6c09-4708-adcb-a3a8814657c9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:45.952868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66217cf1-6ece-49a4-93a7-0866494e7dd9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:45.976093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9df7a9a-1a1a-46d4-80be-f911efa6ab90]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.013162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50a84bd1-cbf8-45ab-aeab-95f685b12ed4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128310, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.037578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1eca3e8-2f2e-4de1-9f62-67e05f2c36f1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.067975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e59b6cc-b2cb-42b2-a720-0fe3023fe9aa]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.102049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99632122-ff01-4903-a6da-eda7a472eb98]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.144137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0784104-901a-47b6-b80a-56b4463fdaaf]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.145092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:46.146488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:46.146960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:28:46.147823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:28:46.177087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:28:46.178120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[372bb3a1-984c-4f5b-bbfa-884e236d9888]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.183968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d748bc4-556c-4c83-823b-6a494d879dbb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.185126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1b4d4e48-4f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:46.234244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1b4d4e48-4f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:46.234918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:46.235313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:46.235846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:46.236794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1b4d4e48-4f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:82:bc:fd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:46.263689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e26ae3c9-6db8-45af-89f7-36a5e9b28944]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.320712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1f2ab6cb-8c6d-4399-85ff-c2924b6b104f]: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:28:46.320712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:28:46.320712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:28:46.320712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:28:46.320712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:28:46.320712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:28:46.320712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:28:46.320712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:28:46.320712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:28:46.320712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:28:46.320712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:28:46.320712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:28:46.320712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:46.322189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f2ab6cb-8c6d-4399-85ff-c2924b6b104f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:46.323385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:46.328482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:28:46.333468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[debca5f8-a5dc-45b5-81d2-42d2f07ce916]: (4, ['qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.334447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:46.334937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:46.335354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:28:46.336027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:46.337305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d60e5490-53ea-48ed-a66b-494276e38164]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.339024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d07cfe67-01c6-41d8-9c8b-fdfeca035fb1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.340752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14af72f9-68e8-4ba0-b079-f9775abf492a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.372971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce48aa0b-7a57-4aa8-a58d-2229127291ac]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 262227, 'reachable_time': 43180, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128328, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.375094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:28:46.375462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4352e607-253b-4a14-8700-bd4300926e6d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:46.376403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersIpV6Test-router-1953029727", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:33Z", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:42Z", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a4a83b6a-52b0-4440-b682-75e82dd825c2", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:bc:fd", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 969, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8::1", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:35Z", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:41Z", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8::/64", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8::1", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6a16d4c-f3b1-41a7-bb8e-a2ad25a45519", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "65bdcd4c-c0dd-489c-bc80-44480a44955a", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d2:85:2d", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 984, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "426fb26e-23ae-4af4-9f3c-9b28bd8b9acf", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8:1::1", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:41Z", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:45Z", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "426fb26e-23ae-4af4-9f3c-9b28bd8b9acf", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8:1::/64", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8:1::1", Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:28:46.380949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id c6f9a587-ce77-4ca7-a332-4b299ea7a872. Wait time elapsed: 2.315 Jul 01 09:28:46.409755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id c6f9a587-ce77-4ca7-a332-4b299ea7a872. Wait time elapsed: 2.315 Jul 01 09:28:46.409755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:47.252419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-d6e5654f-9492-48e8-8d07-f3360356339f tempest-RoutersIpV6AdminTest-1507210859 tempest-RoutersIpV6AdminTest-1507210859-project] Got routers updated notification :['3bc5a891-f1ac-4b91-82d0-93dc4dbe2120'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:47.253664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 3bc5a891-f1ac-4b91-82d0-93dc4dbe2120, action 3, priority 1, update_id fad20820-ab85-4582-af57-6dba7d336eb6. Wait time elapsed: 0.001 Jul 01 09:28:47.253923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 3bc5a891-f1ac-4b91-82d0-93dc4dbe2120, action 3, priority 1, update_id fad20820-ab85-4582-af57-6dba7d336eb6. Wait time elapsed: 0.001 Jul 01 09:28:47.254225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:47.399978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.146s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:47.401445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 3bc5a891-f1ac-4b91-82d0-93dc4dbe2120 was not found. Performing router cleanup Jul 01 09:28:47.402991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc8a5245-b3b6-44fb-b799-f52155d531a7]: (4, ['qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:47.404315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:47.404786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:47.405229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 3bc5a891-f1ac-4b91-82d0-93dc4dbe2120, update_id fad20820-ab85-4582-af57-6dba7d336eb6. Time elapsed: 0.152 Jul 01 09:28:49.864145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.485s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:49.865726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 91f9f498-d760-4ea7-84fe-31c1098b6c91 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:28:49.869299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:49.871051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:49.871570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:49.874702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[475ed7df-846d-4d78-b8e1-c0ac26e92260]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:49.917642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66412436-5aa7-4c0f-83a6-36e0b93c287b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:49.969158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbad3cd0-c3a6-4aaf-a8f0-baad01ed5823]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.018826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36f44ac7-c5d4-4cf2-9160-35cf2e970a60]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128403, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.056948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8764b230-feba-42af-aef2-72df4b110a58]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.093171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba27b122-0eff-4170-a72a-2bfcda602909]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.129944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5abbb020-8b29-45f8-9504-0047419e5c80]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.139979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-1c13c086-8984-4258-abbe-fbea870b82da tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['91f9f498-d760-4ea7-84fe-31c1098b6c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:50.172234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b9b6986-220e-44a5-a16b-0a43056660f0]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.173131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:50.173609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:50.174221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:28:50.174777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:28:50.201930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:28:50.202518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[314d3dcc-5194-4af5-a166-737f8acc2eaa]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.206812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[262ca4ae-f3a0-49c3-a1ac-1d025e7598ea]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.208903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1b4d4e48-4f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:50.247930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1b4d4e48-4f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:50.248930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:50.249508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:50.249887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:50.250288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1b4d4e48-4f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:82:bc:fd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:50.313245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f2ca276-f313-4ec5-afbf-b4e97fca1ea3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.354284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[989b7469-77ec-46c7-8f4d-9af68b11472a]: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:28:50.354284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:28:50.354284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:28:50.354284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:28:50.354284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:28:50.354284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:28:50.354284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:28:50.354284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:28:50.354284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:28:50.354284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:28:50.354284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:28:50.354284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:28:50.354284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[989b7469-77ec-46c7-8f4d-9af68b11472a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:50.355600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:28:50.357700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f43fc55f-8951-4f16-b5c2-1427e73a4872]: (4, ['qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.360294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:50.360294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:50.360294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:28:50.360294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:50.360294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e02dc853-5a40-4909-99ef-e069490e64b2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.370360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6841ef9d-4491-47cd-abea-4ba0e68a766c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.371476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58457633-8866-4fbf-9df6-bd9411aa3db2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.395553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a420401-540a-48e0-96b0-0d6ddf5df41c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 262628, 'reachable_time': 41616, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128422, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.406250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:28:50.406499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56d421e9-c2e7-4c93-8438-f8755419b2aa]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:50.407765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersIpV6Test-router-1953029727", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:33Z", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:47Z", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a4a83b6a-52b0-4440-b682-75e82dd825c2", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:bc:fd", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 969, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8::1", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:35Z", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:47Z", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8::/64", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8::1", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e80cec9-39de-4823-af4a-2954554e87cb", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f952ead8-e792-4f6b-b1d9-292aa643abc4", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d7:c0:bb", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1000, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "9167a61c-3e0e-4fe3-aae3-901d9ac23602", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8:2::1", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:46Z", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:47Z", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9167a61c-3e0e-4fe3-aae3-901d9ac23602", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8:2::/64", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8:2::1", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6a16d4c-f3b1-41a7-bb8e-a2ad25a45519", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "65bdcd4c-c0dd-489c-bc80-44480a44955a", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d2:85:2d", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 984, Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "426fb26e-23ae-4af4-9f3c-9b28bd8b9acf", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8:1::1", Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:28:50.412553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:41Z", Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:45Z", Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "426fb26e-23ae-4af4-9f3c-9b28bd8b9acf", Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8:1::/64", Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8:1::1", Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id 82f7cdce-907d-415a-ac98-32941c767be0. Wait time elapsed: 0.281 Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id 82f7cdce-907d-415a-ac98-32941c767be0. Wait time elapsed: 0.282 Jul 01 09:28:50.430905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:50.614062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-591ad65d-7413-4268-8ed9-526542e4c76d tempest-RoutersIpV6AdminTest-896058136 tempest-RoutersIpV6AdminTest-896058136-project-admin] Got routers updated notification :['3bc5a891-f1ac-4b91-82d0-93dc4dbe2120'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:50.615732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 3bc5a891-f1ac-4b91-82d0-93dc4dbe2120, action 3, priority 1, update_id 519552f3-6b10-4e97-9ec9-b60ca6929233. Wait time elapsed: 0.001 Jul 01 09:28:50.616289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 3bc5a891-f1ac-4b91-82d0-93dc4dbe2120, action 3, priority 1, update_id 519552f3-6b10-4e97-9ec9-b60ca6929233. Wait time elapsed: 0.002 Jul 01 09:28:50.616847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:50.755806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.139s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:50.756111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 3bc5a891-f1ac-4b91-82d0-93dc4dbe2120 was not found. Performing router cleanup Jul 01 09:28:50.757675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd681bc7-90fd-49e7-a9d6-d8f83252781e]: (4, ['qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:50.758209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:50.758500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:50.758763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 3bc5a891-f1ac-4b91-82d0-93dc4dbe2120, update_id 519552f3-6b10-4e97-9ec9-b60ca6929233. Time elapsed: 0.143 Jul 01 09:28:52.520326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c9cbea63-a365-4b68-91e6-d2bdd266f742 tempest-RoutersIpV6AdminTest-1507210859 tempest-RoutersIpV6AdminTest-1507210859-project] Got router deleted notification for 3bc5a891-f1ac-4b91-82d0-93dc4dbe2120 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:28:52.529443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 3bc5a891-f1ac-4b91-82d0-93dc4dbe2120, action 1, priority 1, update_id 2e62d78c-9a26-4049-9545-4bf7044c4fa7. Wait time elapsed: 0.001 Jul 01 09:28:52.530074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 3bc5a891-f1ac-4b91-82d0-93dc4dbe2120, action 1, priority 1, update_id 2e62d78c-9a26-4049-9545-4bf7044c4fa7. Wait time elapsed: 0.009 Jul 01 09:28:52.531195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 3bc5a891-f1ac-4b91-82d0-93dc4dbe2120 was not found. Performing router cleanup Jul 01 09:28:52.533354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bd9a7e7-ff9d-4e48-a6da-9d994f35b549]: (4, ['qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:52.534256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:52.534779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:52.535305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 3bc5a891-f1ac-4b91-82d0-93dc4dbe2120, update_id 2e62d78c-9a26-4049-9545-4bf7044c4fa7. Time elapsed: 0.012 Jul 01 09:28:55.081329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:28:55.558969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-09a3935b-703c-4bee-9413-a7ae93b8b3c3 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['91f9f498-d760-4ea7-84fe-31c1098b6c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:55.698399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.272s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:55.698700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 91f9f498-d760-4ea7-84fe-31c1098b6c91 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:28:55.706710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:28:55.707065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:55.707350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:55.710640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6bf511a7-273b-4ec7-b0c8-ca3cb337612e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:55.740302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4c2b327-07ee-47cf-9d16-d81bfaf7c942]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:55.786070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[993b3755-9cf8-40ef-a811-3442f6b7355b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:55.823010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bdfdd0e8-90e9-4edb-b8cc-3077c838fa84]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128481, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:55.846067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bcabef6-144a-4d9f-8f6a-9815a0d85d14]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:55.902563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02ed76a1-5d98-4edd-9dd3-b5da57649047]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:55.931429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0058d4f-1c77-44ae-ae41-566f5c7a4183]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:55.960530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ffe6e56-e4d2-4041-8b8a-800426ba9275]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:55.964789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:55.965505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:55.966310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:28:55.969233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:28:55.996271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:28:55.996720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca263028-fb21-4179-99e4-bec101fdc9ea]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:56.000743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f26c16e2-e86e-4456-898d-f3ab036416a2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:56.002654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1b4d4e48-4f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:56.034966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1b4d4e48-4f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:56.036727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:56.039131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:56.041096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:56.042447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1b4d4e48-4f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:82:bc:fd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:28:56.097550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5460c2f-7573-492c-9ca6-34ebb2d8f8d4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:56.147400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9db364d9-5d79-40df-8bb5-9bad4b9cc88f]: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:28:56.147400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:28:56.147400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:28:56.147400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:28:56.147400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:28:56.147400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:28:56.147400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:28:56.147400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:28:56.147400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:28:56.147400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:28:56.147400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:28:56.147400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:28:56.147400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:56.149164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9db364d9-5d79-40df-8bb5-9bad4b9cc88f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:56.152148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:56.153909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:28:56.156177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0944e4a8-d646-4a0c-ba95-4e1bb1743b18]: (4, ['qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:56.156177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:56.156177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:56.156432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:28:56.156741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:28:56.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2211571-d482-4a03-85f6-9a7d05de7a81]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:56.164456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c636cfcf-b565-4a94-80d7-401c72e2de9a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:56.165795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df0a33a2-e865-4569-91ae-73d1d59c1cc4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:56.197707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3973864-6ba4-45ec-bdcc-3bd62df416f0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 263208, 'reachable_time': 15592, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128499, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:56.200208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:28:56.200469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69883cbc-4195-4a92-9ccd-bcfcc2bac21f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:28:56.201559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersIpV6Test-router-1953029727", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:33Z", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:53Z", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a4a83b6a-52b0-4440-b682-75e82dd825c2", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:bc:fd", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 969, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8::1", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:35Z", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:53Z", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8::/64", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8::1", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4bede302-600e-4480-b789-b109b88f3971", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "28f402a1-a169-4f42-82fe-5bc39a7145bc", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:14:d3:20", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1010, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "4710a964-672b-4b3b-b9ef-b556bf87b593", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8:3::1", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:52Z", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:53Z", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4710a964-672b-4b3b-b9ef-b556bf87b593", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8:3::/64", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8:3::1", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e80cec9-39de-4823-af4a-2954554e87cb", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f952ead8-e792-4f6b-b1d9-292aa643abc4", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d7:c0:bb", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1000, Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "9167a61c-3e0e-4fe3-aae3-901d9ac23602", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8:2::1", Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:28:56.207831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:46Z", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:54Z", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9167a61c-3e0e-4fe3-aae3-901d9ac23602", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8:2::/64", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8:2::1", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6a16d4c-f3b1-41a7-bb8e-a2ad25a45519", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "65bdcd4c-c0dd-489c-bc80-44480a44955a", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d2:85:2d", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 984, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "426fb26e-23ae-4af4-9f3c-9b28bd8b9acf", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8:1::1", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:41Z", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:45Z", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "426fb26e-23ae-4af4-9f3c-9b28bd8b9acf", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8:1::/64", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8:1::1", Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id 7a10c493-6d2b-4379-872f-d4fcb5d53817. Wait time elapsed: 0.657 Jul 01 09:28:56.222992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id 7a10c493-6d2b-4379-872f-d4fcb5d53817. Wait time elapsed: 0.660 Jul 01 09:28:56.232604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:56.359621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-bb2b4c69-77e4-4e43-b6f1-97bc1fa0a793 tempest-RoutersIpV6AdminTest-1507210859 tempest-RoutersIpV6AdminTest-1507210859-project] Got routers updated notification :['073c12b4-a9fb-47e9-a8d5-32a636eba7b0'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:56.360521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 073c12b4-a9fb-47e9-a8d5-32a636eba7b0, action 3, priority 1, update_id 4eab80c3-4689-498a-9b87-4efc1f982868. Wait time elapsed: 0.001 Jul 01 09:28:56.361697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 073c12b4-a9fb-47e9-a8d5-32a636eba7b0, action 3, priority 1, update_id 4eab80c3-4689-498a-9b87-4efc1f982868. Wait time elapsed: 0.002 Jul 01 09:28:56.362172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:28:56.577618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.215s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:28:56.578445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 073c12b4-a9fb-47e9-a8d5-32a636eba7b0 was not found. Performing router cleanup Jul 01 09:28:56.581790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d411ac3-3a50-4d26-9748-2eb1faece2c7]: (4, ['qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:56.582587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:56.583071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:56.584297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 073c12b4-a9fb-47e9-a8d5-32a636eba7b0, update_id 4eab80c3-4689-498a-9b87-4efc1f982868. Time elapsed: 0.223 Jul 01 09:28:57.125039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-a9016aa6-6540-4cff-a0aa-641dc9910593 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['91f9f498-d760-4ea7-84fe-31c1098b6c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:28:58.731219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-203d9464-22d6-41fb-ae97-7a70909b5400 tempest-RoutersIpV6AdminTest-1507210859 tempest-RoutersIpV6AdminTest-1507210859-project] Got router deleted notification for 073c12b4-a9fb-47e9-a8d5-32a636eba7b0 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:28:58.734732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 073c12b4-a9fb-47e9-a8d5-32a636eba7b0, action 1, priority 1, update_id 1c017d31-f82d-4da9-98fb-940f919d4527. Wait time elapsed: 0.001 Jul 01 09:28:58.735382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 073c12b4-a9fb-47e9-a8d5-32a636eba7b0, action 1, priority 1, update_id 1c017d31-f82d-4da9-98fb-940f919d4527. Wait time elapsed: 0.002 Jul 01 09:28:58.735970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 073c12b4-a9fb-47e9-a8d5-32a636eba7b0 was not found. Performing router cleanup Jul 01 09:28:58.737569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f9fc97b-f3e5-4fab-86a6-577eeffde604]: (4, ['qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:28:58.739614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:28:58.740843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:28:58.741404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 073c12b4-a9fb-47e9-a8d5-32a636eba7b0, update_id 1c017d31-f82d-4da9-98fb-940f919d4527. Time elapsed: 0.007 Jul 01 09:28:59.327140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-0e19afe1-32fb-4c11-b329-680b162e7091 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['91f9f498-d760-4ea7-84fe-31c1098b6c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:02.401819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 6.177s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:02.402538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 91f9f498-d760-4ea7-84fe-31c1098b6c91 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:29:02.403430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:02.403794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:02.404077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:02.405223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea0c5289-1b90-4ce8-8538-a1b7bd05b8c4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.434845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3f98a2c-f5fb-41d1-a3fa-7cb59bcf5f70]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.472822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb626693-a482-4792-a566-8f47d4b9d751]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.502881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-587a2f25-9b98-428f-8a26-7dec045a3a80 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['91f9f498-d760-4ea7-84fe-31c1098b6c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:02.526904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be323a9b-bff8-4b82-bbc0-e755ea1b979c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128612, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.563159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03237b25-16e3-4fe0-bca7-2d0cd4125859]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.598753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5655574-c306-47ce-b26b-db100e141e37]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.632920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e618ca6-682c-425a-8a20-b3a9bcfc3382]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.664206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5280f3d6-9f20-4b3a-baad-b7f779b40970]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.665125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:02.665656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:02.666139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:29:02.666711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:29:02.692231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:29:02.692843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7cf9e561-4792-47ec-8116-fa3db9d20e8f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.696147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42b65a5e-8ba9-43ab-8b3e-8cb359ac4de4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.697406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1b4d4e48-4f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:02.732555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1b4d4e48-4f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:02.733581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:02.734297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:02.736829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:02.737532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1b4d4e48-4f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:82:bc:fd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:02.788842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[805b89db-0941-4fb1-b24f-4b353dd71610]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.829789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8b11383e-d141-4e5d-85c9-4a50414fc8b9]: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:29:02.829789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:29:02.829789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:29:02.829789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:29:02.829789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:29:02.829789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:29:02.829789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:29:02.829789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:29:02.829789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:29:02.829789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:29:02.829789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:29:02.829789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:29:02.829789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:02.830983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b11383e-d141-4e5d-85c9-4a50414fc8b9]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:02.831299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:02.833341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:29:02.835688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[424e1950-b794-4353-9872-eb10131b760f]: (4, ['qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.835688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:02.835688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:02.835688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:29:02.835688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:02.836682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fcb95f4-02e0-4775-acf6-486a1cf803ad]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.837974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d560b1aa-06c0-41f3-b181-fcc779a09735]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.839879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0bc1c4ca-2077-4b71-8efa-18b5f3f4586c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.871352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc466fad-93a0-495f-b1c1-cb32200fa187]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 263878, 'reachable_time': 20868, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128633, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.880958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:29:02.882840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c81a3e7a-4664-4167-95a5-99f29ba92f1c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:02.884118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersIpV6Test-router-1953029727", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:33Z", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:58Z", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a4a83b6a-52b0-4440-b682-75e82dd825c2", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:bc:fd", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 969, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8::1", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:35Z", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:29:00Z", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8::/64", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8::1", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4bede302-600e-4480-b789-b109b88f3971", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "28f402a1-a169-4f42-82fe-5bc39a7145bc", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:14:d3:20", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1010, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "4710a964-672b-4b3b-b9ef-b556bf87b593", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8:3::1", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:52Z", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:29:00Z", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4710a964-672b-4b3b-b9ef-b556bf87b593", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8:3::/64", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8:3::1", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e80cec9-39de-4823-af4a-2954554e87cb", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f952ead8-e792-4f6b-b1d9-292aa643abc4", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d7:c0:bb", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:29:02.889167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1000, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "9167a61c-3e0e-4fe3-aae3-901d9ac23602", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8:2::1", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:46Z", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:54Z", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9167a61c-3e0e-4fe3-aae3-901d9ac23602", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8:2::/64", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8:2::1", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6a16d4c-f3b1-41a7-bb8e-a2ad25a45519", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "65bdcd4c-c0dd-489c-bc80-44480a44955a", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d2:85:2d", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 984, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "426fb26e-23ae-4af4-9f3c-9b28bd8b9acf", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8:1::1", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:41Z", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:45Z", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "426fb26e-23ae-4af4-9f3c-9b28bd8b9acf", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8:1::/64", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8:1::1", Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id 0ceecabc-90d6-4b1d-8007-7213e2ad6ee4. Wait time elapsed: 5.772 Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id 0ceecabc-90d6-4b1d-8007-7213e2ad6ee4. Wait time elapsed: 5.773 Jul 01 09:29:02.904109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:03.332383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c347a4f0-0b84-4169-8938-b68d1a76306f tempest-RoutersIpV6AdminTest-896058136 tempest-RoutersIpV6AdminTest-896058136-project-admin] Got routers updated notification :['d64183e9-6bd8-4075-a925-816bee1b2234'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:03.332935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update d64183e9-6bd8-4075-a925-816bee1b2234, action 3, priority 1, update_id 7e561ddd-b593-43c5-8aaa-29ae77e7155a. Wait time elapsed: 0.000 Jul 01 09:29:03.333180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for d64183e9-6bd8-4075-a925-816bee1b2234, action 3, priority 1, update_id 7e561ddd-b593-43c5-8aaa-29ae77e7155a. Wait time elapsed: 0.001 Jul 01 09:29:03.333466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:03.504370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.171s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:03.506110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router d64183e9-6bd8-4075-a925-816bee1b2234 was not found. Performing router cleanup Jul 01 09:29:03.507235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08e670f7-7504-4520-9f31-57a1ff9456d3]: (4, ['qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:03.508056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:03.509040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:03.509040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for d64183e9-6bd8-4075-a925-816bee1b2234, update_id 7e561ddd-b593-43c5-8aaa-29ae77e7155a. Time elapsed: 0.176 Jul 01 09:29:05.968228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b61a1ac1-d105-4a39-b029-13ed82fa5a3d tempest-RoutersIpV6AdminTest-1507210859 tempest-RoutersIpV6AdminTest-1507210859-project] Got router deleted notification for d64183e9-6bd8-4075-a925-816bee1b2234 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:29:05.970687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update d64183e9-6bd8-4075-a925-816bee1b2234, action 1, priority 1, update_id a184544d-b68d-4fe3-a4e2-f2d71d077a6c. Wait time elapsed: 0.001 Jul 01 09:29:05.977100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for d64183e9-6bd8-4075-a925-816bee1b2234, action 1, priority 1, update_id a184544d-b68d-4fe3-a4e2-f2d71d077a6c. Wait time elapsed: 0.007 Jul 01 09:29:05.977603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router d64183e9-6bd8-4075-a925-816bee1b2234 was not found. Performing router cleanup Jul 01 09:29:05.980274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e6ff7e6-d5fe-4593-adab-21b9bc2c6389]: (4, ['qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:05.981949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:05.984560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.004s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:05.989082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for d64183e9-6bd8-4075-a925-816bee1b2234, update_id a184544d-b68d-4fe3-a4e2-f2d71d077a6c. Time elapsed: 0.014 Jul 01 09:29:06.257682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.359s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:06.259611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 91f9f498-d760-4ea7-84fe-31c1098b6c91 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:29:06.262045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:06.262832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:06.263601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:06.265176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c211975-6836-43db-a054-5a07bfce633f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.296031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f8b5bc0-f7ca-44c5-ba71-3ce260de526f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.342305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d72ff3c-2d13-4123-b965-7d67041a2762]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.392195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0cd8dcdc-cc24-4c06-8b8e-fe12e2303c48]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128642, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.426069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[795e66ac-3199-4999-9a5e-5a79cd37a933]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.450183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e97c8769-b77d-4047-b4ec-83cbefb5dfd2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.484270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e75be6e-083c-4ea9-9597-c843d44f81fa]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.518032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc8e9e95-36e6-41af-bbad-b1759562c353]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.519070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:06.519556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:06.520015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:29:06.524466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:29:06.556854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:29:06.557327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd62b6d6-024e-42ae-8bb0-ad027b5f5707]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.561178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8cefac7b-2121-40c2-8f1b-85c5edebaf26]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.562616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1b4d4e48-4f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:06.613448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1b4d4e48-4f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:06.614516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:06.615116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:06.615804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:06.616415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1b4d4e48-4f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:82:bc:fd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:06.672933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[509f5911-5842-48a9-bc0b-63513c7233c2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4a79cb79-dd5c-4004-b1b6-2d14ad86708c]: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:29:06.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:29:06.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:29:06.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:29:06.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:29:06.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:29:06.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:29:06.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:29:06.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:29:06.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:29:06.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:29:06.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:29:06.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a79cb79-dd5c-4004-b1b6-2d14ad86708c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:06.717611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:29:06.720578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98327830-2bc1-4883-87aa-b2a8e2e829e6]: (4, ['qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.723831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:06.723831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:06.723831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:29:06.723831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:06.727411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[693227fd-d9f8-4e5d-a6ed-fc4358cd9c80]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.727411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aaf537ea-b941-418b-9210-dd0d810058bc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.727411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1ccc38d-10e2-40be-9224-f0e19de50bc4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.752566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d857a3d-03a2-452f-85bd-9f5896418543]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 264265, 'reachable_time': 17300, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128663, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.755567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:29:06.756153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5e1f036-d242-4beb-9a4a-6680934bae28]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:06.758106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersIpV6Test-router-1953029727", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:33Z", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:29:01Z", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a4a83b6a-52b0-4440-b682-75e82dd825c2", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:bc:fd", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 969, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8::1", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:35Z", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:29:04Z", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8::/64", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8::1", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e80cec9-39de-4823-af4a-2954554e87cb", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f952ead8-e792-4f6b-b1d9-292aa643abc4", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d7:c0:bb", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1000, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "9167a61c-3e0e-4fe3-aae3-901d9ac23602", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8:2::1", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:46Z", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:54Z", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9167a61c-3e0e-4fe3-aae3-901d9ac23602", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8:2::/64", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8:2::1", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6a16d4c-f3b1-41a7-bb8e-a2ad25a45519", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "65bdcd4c-c0dd-489c-bc80-44480a44955a", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d2:85:2d", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:29:06.764429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 984, Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "426fb26e-23ae-4af4-9f3c-9b28bd8b9acf", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8:1::1", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:41Z", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:45Z", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "426fb26e-23ae-4af4-9f3c-9b28bd8b9acf", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8:1::/64", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8:1::1", Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id 5d7d3049-6721-4e59-b642-d187aa10b568. Wait time elapsed: 7.437 Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id 5d7d3049-6721-4e59-b642-d187aa10b568. Wait time elapsed: 7.438 Jul 01 09:29:06.784839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:06.815888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-51cca25c-2eea-4ad9-bb2c-6e00a94a676d tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['91f9f498-d760-4ea7-84fe-31c1098b6c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:09.785221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.019s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:09.786223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 91f9f498-d760-4ea7-84fe-31c1098b6c91 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:29:09.787705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:09.788193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:09.789011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:09.790862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[029e3e34-5128-47dd-b17e-50c3893dfba8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:09.826334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[612871fd-3388-40d8-a257-0a9d1d7d95ff]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:09.867578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19e10122-b2d3-4ee7-89b5-47b89ae75dbe]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:09.932022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-e651228f-4dff-40c5-95a0-8d420d8d9598 tempest-RoutersIpV6AdminTest-896058136 tempest-RoutersIpV6AdminTest-896058136-project-admin] Got routers updated notification :['f38cdacf-054b-41f3-b0d9-5611dc5609e3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:09.940353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19a0368e-ea37-444b-8e3c-ab998b2f0548]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128732, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:09.946322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f38cdacf-054b-41f3-b0d9-5611dc5609e3, action 3, priority 1, update_id 685a9655-b7fd-4c16-893f-2a1b01959258. Wait time elapsed: 0.014 Jul 01 09:29:09.946685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f38cdacf-054b-41f3-b0d9-5611dc5609e3, action 3, priority 1, update_id 685a9655-b7fd-4c16-893f-2a1b01959258. Wait time elapsed: 0.014 Jul 01 09:29:09.947289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:09.989144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c94cbaa-50f9-4862-a8a8-26a50684b214]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.027233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ec0f7c0-c224-434b-ae93-bd883e596593]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.072627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9722c5ca-76fc-400d-b453-b833cb4a48b9]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.114994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[607a78a7-5e62-42fa-b60b-44e01b63307b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.116172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:10.116589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:10.116958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:29:10.117338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:29:10.148920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:29:10.149411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09a9ac3b-3b73-4b9a-90e8-7316fd30a0cb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.152207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20b8b44a-0077-42e0-9048-89f4bc08e9ba]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.153332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1b4d4e48-4f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:10.188140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1b4d4e48-4f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:10.191222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:10.192105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:10.192659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:10.193432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1b4d4e48-4f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:82:bc:fd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:10.232953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68feb7ac-e9bf-4b05-bed6-928885e2e7f2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.262571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.315s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:10.263159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f38cdacf-054b-41f3-b0d9-5611dc5609e3 was not found. Performing router cleanup Jul 01 09:29:10.265460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df85ab9b-b445-4e56-a6c1-4a16e7d098b0]: (4, ['qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.267288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:10.268673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:10.269144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f38cdacf-054b-41f3-b0d9-5611dc5609e3, update_id 685a9655-b7fd-4c16-893f-2a1b01959258. Time elapsed: 0.323 Jul 01 09:29:10.288097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3561ee44-bc39-40b9-b5c8-c0ef4207bb17]: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:29:10.288097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:29:10.288097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:29:10.288097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:29:10.288097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:29:10.288097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:29:10.288097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:29:10.288097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:29:10.288097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:29:10.288097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:29:10.288097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:29:10.288097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:29:10.288097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:10.289432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3561ee44-bc39-40b9-b5c8-c0ef4207bb17]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:10.290041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:10.294364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:29:10.298784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45df7bcb-4e67-4199-be4c-9cdc72f76e49]: (4, ['qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.299703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:10.300085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:10.300342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:29:10.300739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:10.301941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52a62089-40f7-4e81-bb8c-befb43500973]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.302919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b587817-5452-4854-b01c-eb8a619940b9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.303854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e66f5840-1558-4116-9d4b-7707876d3ab5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.341896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[788ecf60-f384-4339-a7c9-c21c80f1d8b1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 264618, 'reachable_time': 43200, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128751, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.344067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:29:10.344314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[941af8e2-a4c4-4a48-b633-cb3f73cb2f1a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:10.346163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersIpV6Test-router-1953029727", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:33Z", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:29:05Z", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a4a83b6a-52b0-4440-b682-75e82dd825c2", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:bc:fd", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 969, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8::1", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:35Z", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:29:07Z", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8::/64", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8::1", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6a16d4c-f3b1-41a7-bb8e-a2ad25a45519", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "65bdcd4c-c0dd-489c-bc80-44480a44955a", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d2:85:2d", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 984, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "426fb26e-23ae-4af4-9f3c-9b28bd8b9acf", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8:1::1", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:41Z", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:28:45Z", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "426fb26e-23ae-4af4-9f3c-9b28bd8b9acf", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8:1::/64", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8:1::1", Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:29:10.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id a9924b43-c4c7-4b03-bb28-36388bcb5dd7. Wait time elapsed: 7.844 Jul 01 09:29:10.369523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id a9924b43-c4c7-4b03-bb28-36388bcb5dd7. Wait time elapsed: 7.844 Jul 01 09:29:10.369523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:11.584522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-15fafdc0-7e3c-43db-a6d6-d78dfe5f1336 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['91f9f498-d760-4ea7-84fe-31c1098b6c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:12.293659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8f99e30b-160b-4464-b132-d98242f6ce12 tempest-RoutersIpV6AdminTest-1507210859 tempest-RoutersIpV6AdminTest-1507210859-project] Got router deleted notification for f38cdacf-054b-41f3-b0d9-5611dc5609e3 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:29:12.295052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f38cdacf-054b-41f3-b0d9-5611dc5609e3, action 1, priority 1, update_id c9c1fc91-7b25-49bb-b5de-2126305e67f1. Wait time elapsed: 0.001 Jul 01 09:29:12.295479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f38cdacf-054b-41f3-b0d9-5611dc5609e3, action 1, priority 1, update_id c9c1fc91-7b25-49bb-b5de-2126305e67f1. Wait time elapsed: 0.001 Jul 01 09:29:12.295994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f38cdacf-054b-41f3-b0d9-5611dc5609e3 was not found. Performing router cleanup Jul 01 09:29:12.297248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd08ee9d-6421-4a44-8852-df41521857df]: (4, ['qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:12.298997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:12.299268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:12.299563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f38cdacf-054b-41f3-b0d9-5611dc5609e3, update_id c9c1fc91-7b25-49bb-b5de-2126305e67f1. Time elapsed: 0.004 Jul 01 09:29:13.554240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.205s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:13.557261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 91f9f498-d760-4ea7-84fe-31c1098b6c91 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:29:13.565032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:13.566171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:13.566762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:13.569673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ae7eba6-80da-42fb-bde9-05b04f110458]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:13.601239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5a7c575-1dfd-4d69-9c63-fe3c2e159109]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:13.636401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1ca3050-8671-46fd-a01d-5557eda32955]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:13.721362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af85c1e6-ee81-460e-8a4f-a6f1dfa1d81b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128754, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:13.772066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8dc865d5-ecab-42e1-aa11-a340d5db4a9d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:13.828549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e2305cd-378c-4652-805a-eb5d89db87b2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:13.863473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79a19d05-364c-4886-b02c-35c9a0f3aecb]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:13.911638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19f15ba9-7525-41ff-84c6-9c9007f1ad78]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:13.912204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:13.912494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:13.912757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:29:13.913033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:29:13.959460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:29:13.960033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f384e677-452d-49c6-ad5c-ae4ac46bd19c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:13.964488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9fcff51e-b556-4d9a-aff5-b296918c9b8a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:13.965882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1b4d4e48-4f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:14.008075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1b4d4e48-4f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:14.008743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:14.009133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:14.009516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:14.009897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1b4d4e48-4f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:82:bc:fd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:14.053537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0c9e081-1953-4db6-bae1-0b59874530d8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:14.125294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[71193bde-8895-45a7-a4f8-dabd61cb523e]: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:29:14.125294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:29:14.125294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:29:14.125294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:29:14.125294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:29:14.125294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:29:14.125294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:29:14.125294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:29:14.125294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:29:14.125294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:29:14.125294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:29:14.125294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:29:14.125294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:14.126589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71193bde-8895-45a7-a4f8-dabd61cb523e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:14.126802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:14.129664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:29:14.145517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8839ba0-47ab-4384-bc77-e1de6434a4d8]: (4, ['qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:14.146254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:14.146686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:14.147013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:29:14.147559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:14.149571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cdb22c68-b162-42a5-8704-df2ed2379eaf]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:14.150548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5056c45f-3ff4-43d3-92a4-e006a725178f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:14.151601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4ba1a75-1c4d-4dec-aabe-6518f31ac540]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:14.186538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40d38370-a9b6-43f4-ae0d-3dea897d8868]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 264998, 'reachable_time': 20500, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128801, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:14.189235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:29:14.189735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[519983ab-c36a-4520-b213-350a5f1d1125]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:14.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersIpV6Test-router-1953029727", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:33Z", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:29:10Z", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a4a83b6a-52b0-4440-b682-75e82dd825c2", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:bc:fd", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 969, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8::1", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:35Z", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:29:09Z", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8::/64", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8::1", Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:29:14.197696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:29:14.208240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id 6ba8a93f-2b47-4dfb-bf62-7e4120a7b404. Wait time elapsed: 7.390 Jul 01 09:29:14.208240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id 6ba8a93f-2b47-4dfb-bf62-7e4120a7b404. Wait time elapsed: 7.390 Jul 01 09:29:14.208240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:15.072216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:15.072907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:15.072907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:15.073081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:15.297247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.090s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:15.298373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 91f9f498-d760-4ea7-84fe-31c1098b6c91 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:29:15.299577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:15.300121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:15.300612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:15.301937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb1010f3-7e3a-4984-8f0c-dae58922df1a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.329588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32252e50-a74e-4326-acd4-f0b460d94f19]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.357152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ec8ba6f-b62d-4004-a6d7-dc6e3b386b2e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.393835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc8b990b-b955-4501-a755-7bd6b411ebff]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128824, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.421884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5762048-b794-47b7-818c-df1d2c9b40a2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.468686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da5174c7-58cf-466d-9d8c-fc6bfda023d6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.515003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85da5413-c943-452b-93d9-c478caaeef03]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.556990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ca9b570-2973-4dfe-a43f-7ffff44350e3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.559766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:15.561506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:15.562103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:29:15.563681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:29:15.611168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:29:15.612415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c39ec33-2c56-40ce-aa6c-33156647127a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.619706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d95446ec-fdb8-4f7b-9bc3-c707c1bd09b8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.621628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1b4d4e48-4f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:15.661085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1b4d4e48-4f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:15.663753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:15.665537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:15.666741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:15.667773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1b4d4e48-4f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:82:bc:fd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:15.737673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52c81e9b-afd4-40e7-9cd2-fdd2d1ebe830]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.785674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c0a45138-0089-4344-966e-026151e5416d]: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:29:15.785674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:29:15.785674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:29:15.785674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:29:15.785674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:29:15.785674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:29:15.785674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:29:15.785674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:29:15.785674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:29:15.785674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:29:15.785674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:29:15.785674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:29:15.785674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0a45138-0089-4344-966e-026151e5416d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:15.786880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:29:15.789274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19db5816-3680-4131-a5e0-407ff74b51b1]: (4, ['qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.793003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:15.793003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:15.793003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:29:15.793003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:15.794701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1dc3d01f-2b51-48c6-9059-020c6f7be053]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.795520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8b755b4-1c94-4d71-8010-2644c46a4c4a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.796378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89202418-f77e-414c-96ed-0d8ba9e140af]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.832437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-27bbb693-d3e8-45e4-ae6a-de9df413fc16 tempest-RoutersIpV6AdminTest-1507210859 tempest-RoutersIpV6AdminTest-1507210859-project] Got routers updated notification :['9e9dd644-7826-41ac-9457-2fb4de6b5f0d'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:15.833368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e9dd644-7826-41ac-9457-2fb4de6b5f0d, action 3, priority 1, update_id d79b92bf-2913-49c7-b1fa-6f36adf9b919. Wait time elapsed: 0.001 Jul 01 09:29:15.833613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e9dd644-7826-41ac-9457-2fb4de6b5f0d, action 3, priority 1, update_id d79b92bf-2913-49c7-b1fa-6f36adf9b919. Wait time elapsed: 0.001 Jul 01 09:29:15.833901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:15.838436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5b85d1e-7a0c-4a53-b2d6-53fb5804d9b8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 265165, 'reachable_time': 27008, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128842, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.845563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:29:15.845820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b063417-58ca-47d3-b2df-68c124e93051]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:15.846914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersIpV6Test-router-1953029727", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:33Z", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:29:10Z", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a4a83b6a-52b0-4440-b682-75e82dd825c2", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:bc:fd", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 969, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8::1", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:35Z", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:29:14Z", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8::/64", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8::1", Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id a0c71056-8032-4295-8ad7-4471d89bb1bb. Wait time elapsed: 4.262 Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id a0c71056-8032-4295-8ad7-4471d89bb1bb. Wait time elapsed: 4.262 Jul 01 09:29:15.850645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:16.430644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.597s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:16.431291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9e9dd644-7826-41ac-9457-2fb4de6b5f0d was not found. Performing router cleanup Jul 01 09:29:16.432698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21b51a46-a530-4572-85c1-dd55a9355c4e]: (4, ['qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:16.433879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:16.434270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:16.434598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9e9dd644-7826-41ac-9457-2fb4de6b5f0d, update_id d79b92bf-2913-49c7-b1fa-6f36adf9b919. Time elapsed: 0.601 Jul 01 09:29:16.862434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.014s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:16.862718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 91f9f498-d760-4ea7-84fe-31c1098b6c91 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:29:16.863686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:16.863995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:16.864278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:16.865579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5f77415-bd1a-4865-816b-9547d960abff]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:16.890064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd79c06e-dae6-4e4b-9e69-3921dafe2a09]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:16.920246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd16ffba-0608-4856-90b8-be6d9e1c5942]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:16.991846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a3c2021-55ee-4cd6-8edb-89c9fc8a35eb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128854, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.060013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[740c98c0-6871-4830-9a59-be19452e1d87]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.097772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15dd9403-0200-4c61-b97f-27955c9c551a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.128025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a61911f-d170-403a-a422-bad5ed925ac6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.166213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51d7473f-2603-48f0-8e3b-91721fd3c456]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.166782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:17.167260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:17.167696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:29:17.168126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:29:17.196352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:29:17.196597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e012a020-ee38-4d9e-8cdf-eda217525205]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.199440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d98cc73-ee09-4ac0-915f-1d5a8dd9465f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.201006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1b4d4e48-4f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:17.236455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1b4d4e48-4f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:17.237587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:17.238621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:17.239220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1b4d4e48-4f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:17.239889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1b4d4e48-4f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:82:bc:fd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:29:17.288418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87ce0c50-7c18-4444-aa4f-7fa72b1e1368]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.331882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[fd961b8a-0fd0-48a8-bcf9-81d16c0210ef]: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:29:17.331882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:29:17.331882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:29:17.331882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:29:17.331882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:29:17.331882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:29:17.331882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:29:17.331882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:29:17.331882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:29:17.331882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:29:17.331882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:29:17.331882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:29:17.331882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:17.332966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd961b8a-0fd0-48a8-bcf9-81d16c0210ef]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:17.333104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:17.337123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:29:17.341442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b2a4ac7-9aa0-48fb-8754-09137fdd5131]: (4, ['qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.342674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:17.343275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:17.343853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:29:17.344557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:17.346497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c67e5a67-f27d-4714-ab59-c2ce3c9dd3e8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.348282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c83eb15d-3e8f-4c1f-b49f-2d6b97148512]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.349669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b97bf96f-7dbf-4262-b1b0-ad7c9327e3b2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.386405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[480f5260-3546-4d65-afe3-2364333727e5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 265325, 'reachable_time': 39400, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128886, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.400348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:29:17.400965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fea8169-db57-4206-977f-8ddf0f4fb4eb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 91f9f498-d760-4ea7-84fe-31c1098b6c91: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1b4d4e48-4f not found in namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91. Jul 01 09:29:17.402486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersIpV6Test-router-1953029727", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:33Z", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:29:10Z", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1b4d4e48-4fcb-4fa4-a6a2-6a7b43c541e5", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a4a83b6a-52b0-4440-b682-75e82dd825c2", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:82:bc:fd", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "91f9f498-d760-4ea7-84fe-31c1098b6c91", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 969, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "2001:db8::1", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 64 Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:28:35Z", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:29:15Z", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ff7fdb6df28b4a6787120539e7356162", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b55d1339-a9db-4b53-9ba1-0f4386f83644", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "2001:db8::/64", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "2001:db8::1", Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:29:17.406778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:29:17.418948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 2, update_id acc46d94-5045-442a-934d-1c70f6ede10b. Wait time elapsed: 39.315 Jul 01 09:29:17.418948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 2, update_id acc46d94-5045-442a-934d-1c70f6ede10b. Wait time elapsed: 39.315 Jul 01 09:29:17.419994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:18.587926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-2071a3cc-56ca-48a9-8546-89b091934694 tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['91f9f498-d760-4ea7-84fe-31c1098b6c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:18.687062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.269s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:18.687687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 91f9f498-d760-4ea7-84fe-31c1098b6c91 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:29:18.692427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:18.693065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:18.694118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:18.696834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54323974-392e-46f1-be83-a0a63f1fab32]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:18.725857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b8e34d8-6d2d-4ecd-99f8-312e49e1e687]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:18.756473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24fd0547-5b00-4bc0-a37c-c4ac97c158c5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:18.811692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16b13fea-5608-4425-9e41-c0ada0550b47]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 128905, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:18.835154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[726baed1-fecc-45ae-a14a-5e6f90c32c1c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:18.867889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3a9f946-bc95-4dbb-abf2-cdcc58618e4e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:18.897430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3883e059-62f3-43e0-8a44-3e524343436d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:18.949888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a02973f2-b0be-4931-adbc-a27df0775571]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:18.950706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:18.953395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:18.954349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:29:18.989859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab536b5b-bc3b-4e00-a8f7-542566c9269a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 265507, 'reachable_time': 27372, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128912, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:18.991193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:18.991672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:18.992264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:18.992691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:29:19.021263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1d41b6f-9a48-4fd5-9fe1-ca69c7978ba1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 265507, 'reachable_time': 27372, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128914, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:19.023165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:19.023953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:29:19.062123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1b1b761-38a7-4608-b8be-7869def7d067]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:19.270730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bdb831f1-3eff-4827-b961-3628037fd712]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:19.307111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[692ea1b6-73a7-49c5-bee4-c66e07016a6c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:19.493805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48cefdc5-e86c-42ef-81a6-68b0d7b9585f]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:19.494949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:29:19.495472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:19.495908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:19.496353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" released by "process_external" :: held 0.504s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:29:19.497503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:19.497929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" acquired by "process_address_scope" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:29:19.498482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:19.499542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:29:19.506904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-232c90c4-ace4-49ea-bd67-0fb8ec4310b6 tempest-RoutersIpV6AdminTest-1507210859 tempest-RoutersIpV6AdminTest-1507210859-project] Got routers updated notification :['9e9dd644-7826-41ac-9457-2fb4de6b5f0d'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:19.507841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e9dd644-7826-41ac-9457-2fb4de6b5f0d, action 3, priority 1, update_id 74af596b-7c0d-44c4-b781-3a673ba48044. Wait time elapsed: 0.001 Jul 01 09:29:19.508087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e9dd644-7826-41ac-9457-2fb4de6b5f0d, action 3, priority 1, update_id 74af596b-7c0d-44c4-b781-3a673ba48044. Wait time elapsed: 0.001 Jul 01 09:29:19.508386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:19.534097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[708d87df-ee14-477e-9edc-1955e3981370]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:19.590022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e536927-1e7e-45f3-98ab-f7b835579cab]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:19.630087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f89f437f-90b8-44a4-b764-5c79a094b106]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:19.666038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.158s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:19.666517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9e9dd644-7826-41ac-9457-2fb4de6b5f0d was not found. Performing router cleanup Jul 01 09:29:19.668177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c691d24c-4868-420a-9f29-d12e33c5c31f]: (4, ['qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:19.668679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:19.668943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:19.669369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9e9dd644-7826-41ac-9457-2fb4de6b5f0d, update_id 74af596b-7c0d-44c4-b781-3a673ba48044. Time elapsed: 0.162 Jul 01 09:29:19.691378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d68a4ab2-1553-4727-a672-8d203506404a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:19.691866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:29:19.692316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:19.692552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:19.692834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" released by "process_address_scope" :: held 0.195s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:29:19.693449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:19.694032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:19.694277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:29:19.694626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:19.695063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:29:19.735476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a1f98aa-765f-4c81-9a4b-ba96ea8ba8b6]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:19.835868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6fafb1f-fec5-44c3-85d2-e9cc3ea71a6f]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:19.929538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7090bbb3-d7aa-426b-b18a-1b98f808e834]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:19 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:29:19 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:20.008396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4da6e3b3-3420-47d8-80e9-f1d85233e3f2]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:20.009088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:29:20.009573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:20.010191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:20.010986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" released by "apply_metadata_nat_rules" :: held 0.316s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:29:20.011854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-91f9f498-d760-4ea7-84fe-31c1098b6c91 Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.haproxy Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 91f9f498-d760-4ea7-84fe-31c1098b6c91 Jul 01 09:29:20.012359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:29:20.014731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/91f9f498-d760-4ea7-84fe-31c1098b6c91.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:29:20.049769 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-91f9f498-d760-4ea7-84fe-31c1098b6c91[128956]: Proxy listener started. Jul 01 09:29:20.061232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 91f9f498-d760-4ea7-84fe-31c1098b6c91 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:29:20.061517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, update_id acc46d94-5045-442a-934d-1c70f6ede10b. Time elapsed: 2.644 Jul 01 09:29:20.061803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id 066625e7-8206-420e-b939-5f617e4e5273. Wait time elapsed: 1.473 Jul 01 09:29:20.062056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id 066625e7-8206-420e-b939-5f617e4e5273. Wait time elapsed: 1.473 Jul 01 09:29:20.062369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:20.696618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-da04ba7a-b67b-4779-93cc-d72c7376cd11 tempest-RoutersIpV6AdminTest-1507210859 tempest-RoutersIpV6AdminTest-1507210859-project] Got router deleted notification for 9e9dd644-7826-41ac-9457-2fb4de6b5f0d {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:29:20.698952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e9dd644-7826-41ac-9457-2fb4de6b5f0d, action 1, priority 1, update_id f48303e4-f6b0-496f-88fd-951d1e8defa9. Wait time elapsed: 0.001 Jul 01 09:29:20.699470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e9dd644-7826-41ac-9457-2fb4de6b5f0d, action 1, priority 1, update_id f48303e4-f6b0-496f-88fd-951d1e8defa9. Wait time elapsed: 0.001 Jul 01 09:29:20.699950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9e9dd644-7826-41ac-9457-2fb4de6b5f0d was not found. Performing router cleanup Jul 01 09:29:20.701269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddd5895c-f594-416b-a0da-58aaabbef311]: (4, ['qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:20.704789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:20.705401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:20.705977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9e9dd644-7826-41ac-9457-2fb4de6b5f0d, update_id f48303e4-f6b0-496f-88fd-951d1e8defa9. Time elapsed: 0.007 Jul 01 09:29:20.739806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.677s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:20.741012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 91f9f498-d760-4ea7-84fe-31c1098b6c91 info in cache, will do the router update action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:637}} Jul 01 09:29:20.741370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks [] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_update {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:20.741622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:29:20.766167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04f3d794-c1a0-4f8c-920b-31c9ea13bae4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 265507, 'reachable_time': 27372, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128961, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:20.767015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:20.767294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:20.767742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:20.768001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:29:20.795166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0023a97-e801-4a4a-852e-7f7c4a5ecbd9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 265507, 'reachable_time': 27372, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128962, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:20.796028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:20.796563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:29:20.830169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cf5d944-c75e-4588-951a-cf6e74e2170c]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-INPUT -m mark --mark 0x1/0xffff -j ACCEPT\n-A neutron-l3-agent-INPUT -p tcp -m tcp --dport 9697 -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:20.862683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[318a15c1-2779-4c97-8916-cb9d1040c268]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-INPUT -m mark --mark 0x1/0xffff -j ACCEPT\n-A neutron-l3-agent-INPUT -p tcp -m tcp --dport 9697 -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:20.865185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 0 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:29:20.865729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:20.866204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:20.866736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" released by "process_external" :: held 0.099s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:29:20.867476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:20.867957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" acquired by "process_address_scope" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:29:20.868515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:20.869185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:29:20.913272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eefb29cd-2598-49da-92c3-ab0edb2d6f55]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-INPUT -m mark --mark 0x1/0xffff -j ACCEPT\n-A neutron-l3-agent-INPUT -p tcp -m tcp --dport 9697 -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:20.963106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f2f082c-9abf-4a39-9425-72f1537e3155]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:20 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-INPUT -m mark --mark 0x1/0xffff -j ACCEPT\n-A neutron-l3-agent-INPUT -p tcp -m tcp --dport 9697 -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:29:20 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:20.966048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 0 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:29:20.966670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:20.967291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:20.967827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" released by "process_address_scope" :: held 0.100s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:29:20.968482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_updated-8767249873071', 'neutron.agent.linux.pd.update_router-8767249872375'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), after_update {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:20.969041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.update_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:20.969744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.update_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:20.970269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 91f9f498-d760-4ea7-84fe-31c1098b6c91 update action. {{(pid=106231) update_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:65}} Jul 01 09:29:20.971596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, update_id 066625e7-8206-420e-b939-5f617e4e5273. Time elapsed: 0.910 Jul 01 09:29:21.955220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-cf995bc4-4b68-4fdd-83a1-e9a1c17ec2ce tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['91f9f498-d760-4ea7-84fe-31c1098b6c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:21.956996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id c52b309d-56b1-42a0-ab5e-80d138106250. Wait time elapsed: 0.001 Jul 01 09:29:21.957804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 3, priority 1, update_id c52b309d-56b1-42a0-ab5e-80d138106250. Wait time elapsed: 0.001 Jul 01 09:29:21.958306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:22.428284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-dde56fa4-619a-467b-8098-43b08dd746bd tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got router deleted notification for 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:29:22.616409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.658s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:22.616789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 91f9f498-d760-4ea7-84fe-31c1098b6c91 info in cache, will do the router update action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:637}} Jul 01 09:29:22.617454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks [] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_update {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:22.617950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:29:22.661252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c25018ee-ed4b-4ec4-8b7b-cc93871ed67b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 265507, 'reachable_time': 27372, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128967, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:22.663696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:22.664267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:22.664832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:22.665886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:29:22.695902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77c1e23f-db5d-407d-bc26-552d8cbfded7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 265507, 'reachable_time': 27372, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128968, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:22.697452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:22.698138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:29:22.725070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7790663f-8ad4-4c13-bf9b-af21b6d0b70c]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-INPUT -m mark --mark 0x1/0xffff -j ACCEPT\n-A neutron-l3-agent-INPUT -p tcp -m tcp --dport 9697 -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:22.772119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d8debf4-12f9-4d45-99c8-1ca849b7fe6b]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-INPUT -m mark --mark 0x1/0xffff -j ACCEPT\n-A neutron-l3-agent-INPUT -p tcp -m tcp --dport 9697 -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:22.774177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 0 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:29:22.774467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:22.774760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:22.775030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" released by "process_external" :: held 0.110s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:29:22.775542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:22.775795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:29:22.776414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:22.777018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:29:22.810171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06a75e97-07c4-4390-808c-eccde5647302]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-INPUT -m mark --mark 0x1/0xffff -j ACCEPT\n-A neutron-l3-agent-INPUT -p tcp -m tcp --dport 9697 -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:22.831159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-d97d43ca-b51c-4ddc-81bd-bc5e16c98dc7 tempest-PortsTestJSON-806010939 tempest-PortsTestJSON-806010939-project] Got routers updated notification :['50ac41ef-2def-4c27-ba1b-8893b0287e65'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:22.833569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 50ac41ef-2def-4c27-ba1b-8893b0287e65, action 3, priority 1, update_id 9d6386ef-cfc3-4cc3-85b2-8bac995a768f. Wait time elapsed: 0.001 Jul 01 09:29:22.834748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 50ac41ef-2def-4c27-ba1b-8893b0287e65, action 3, priority 1, update_id 9d6386ef-cfc3-4cc3-85b2-8bac995a768f. Wait time elapsed: 0.002 Jul 01 09:29:22.835657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:22.869015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbd793c5-3b22-4db3-8cd9-e01ba4d93c12]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:22 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-INPUT -m mark --mark 0x1/0xffff -j ACCEPT\n-A neutron-l3-agent-INPUT -p tcp -m tcp --dport 9697 -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:29:22 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:22.873685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 0 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:29:22.874297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:22.874857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:22.875464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91" released by "process_address_scope" :: held 0.100s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:29:22.876366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_updated-8767249873071', 'neutron.agent.linux.pd.update_router-8767249872375'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), after_update {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:22.877051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.update_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:22.877524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.update_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:22.878427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 91f9f498-d760-4ea7-84fe-31c1098b6c91 update action. {{(pid=106231) update_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:65}} Jul 01 09:29:22.879958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, update_id c52b309d-56b1-42a0-ab5e-80d138106250. Time elapsed: 0.923 Jul 01 09:29:22.880478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 1, priority 1, update_id eaa7ed34-a4bb-4899-b79a-a2230689d650. Wait time elapsed: 0.452 Jul 01 09:29:22.881103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 91f9f498-d760-4ea7-84fe-31c1098b6c91, action 1, priority 1, update_id eaa7ed34-a4bb-4899-b79a-a2230689d650. Wait time elapsed: 0.452 Jul 01 09:29:22.881851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 91f9f498-d760-4ea7-84fe-31c1098b6c91 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:29:22.882448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:22.908657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0521c1b4-8769-4995-bf9f-dcb1ba0ccb25]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:22.914285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a61ec52-4c30-4ffa-864d-250bf55f7852]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:22.915646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[883f2907-af4a-4f23-95e0-2c4d2f29b363]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:22.917143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:29:22.920218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[876dc2f0-a06a-4e16-9e17-c78f9a9617bb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:22.961090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe8342dd-427c-4daf-b535-d8fda1ae7b75]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 265507, 'reachable_time': 27372, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128974, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:22.964013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:22.964873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:23.001993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3079d38-317f-49dd-a644-4a5c798c848d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 265507, 'reachable_time': 27372, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128975, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:23.003343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:29:23.004074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:23.004636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/91f9f498-d760-4ea7-84fe-31c1098b6c91.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:23.005079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:29:23.010220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 91f9f498-d760-4ea7-84fe-31c1098b6c91 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:29:23.013766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c2506b8-c4e3-42ee-8cff-a13db12e8a0a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:23.051514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[227472f9-538e-4901-b728-5cb2ea38480f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 265507, 'reachable_time': 27372, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 128976, 'error': None, 'target': 'qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:23.057175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-91f9f498-d760-4ea7-84fe-31c1098b6c91 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:29:23.057281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c280eb4d-2e8a-4b72-a33a-56110f5459d7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:23.057691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 91f9f498-d760-4ea7-84fe-31c1098b6c91 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:29:23.058028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (91f9f498-d760-4ea7-84fe-31c1098b6c91), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:23.058872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:23.059174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:23.059629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:23.060534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:23.060990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 91f9f498-d760-4ea7-84fe-31c1098b6c91, update_id eaa7ed34-a4bb-4899-b79a-a2230689d650. Time elapsed: 0.181 Jul 01 09:29:23.283883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.447s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:23.287103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 50ac41ef-2def-4c27-ba1b-8893b0287e65 was not found. Performing router cleanup Jul 01 09:29:23.288853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04833ffc-2c85-4a81-b7da-e71bfaf2d9d9]: (4, ['qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:23.290270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:23.290763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:23.291229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 50ac41ef-2def-4c27-ba1b-8893b0287e65, update_id 9d6386ef-cfc3-4cc3-85b2-8bac995a768f. Time elapsed: 0.458 Jul 01 09:29:23.615086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-fd76ff17-6586-41e1-91be-d561885273cb tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got routers updated notification :['fc69ce97-e3e0-4d15-8e65-00ab7def92c1'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:23.622331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fc69ce97-e3e0-4d15-8e65-00ab7def92c1, action 3, priority 1, update_id dab1f34a-4abd-4e15-a235-3ec080c34a37. Wait time elapsed: 0.006 Jul 01 09:29:23.633282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fc69ce97-e3e0-4d15-8e65-00ab7def92c1, action 3, priority 1, update_id dab1f34a-4abd-4e15-a235-3ec080c34a37. Wait time elapsed: 0.017 Jul 01 09:29:23.633639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:24.303093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.669s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:24.303738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router fc69ce97-e3e0-4d15-8e65-00ab7def92c1 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:29:24.305039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (fc69ce97-e3e0-4d15-8e65-00ab7def92c1), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:24.305751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:24.306308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:24.322317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-eb432b97-7932-4099-9ffc-63c28b3f32eb tempest-RoutersIpV6Test-405583564 tempest-RoutersIpV6Test-405583564-project] Got router deleted notification for fc69ce97-e3e0-4d15-8e65-00ab7def92c1 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:29:24.323571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d708bc1-f217-4bdd-bb60-525a1434e449]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:24.354006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74ecd14f-94ae-4da1-96a3-da1bb7239056]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:24.392611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1de1ab30-e1e3-4491-9ae1-88015e31022c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:24.450683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5eb3ec52-55b1-41a7-a4d7-09c855ee9fbd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 129012, 'error': None, 'target': 'qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:24.488970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cda3a90-5ba2-470a-963b-b9d809800ca3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:24.538910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdb7df0f-303f-43d4-9558-bec1c7f25c90]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:24.565594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61412f52-534d-4545-85c3-7eddc4fb92e7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:24.635377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bba26aa5-9de1-4531-a1eb-eedb2b09a31c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:24.638792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:24.666783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.029s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:24.667341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router fc69ce97-e3e0-4d15-8e65-00ab7def92c1 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:29:24.686556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df56916d-2a23-4743-87c9-4457d1729f3a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 266071, 'reachable_time': 27836, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 129034, 'error': None, 'target': 'qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:24.687662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:24.688699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:24.689396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:24.689823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:29:24.744919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[781e21f2-8c72-437f-8ef8-1b78cabbd08b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 266071, 'reachable_time': 27836, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 129035, 'error': None, 'target': 'qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:24.746008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:24.746842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:29:24.795810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[593eb799-1903-4c8b-9cd0-3920d2d51d8d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.044132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35cf62ef-1c6b-403f-bb2c-12eec7a86102]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.089440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb3713a4-e171-4970-b923-2746b4d4851c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.274730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da2b5eda-d825-473c-bbb7-2689b0abd6f7]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.276068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:29:25.276621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:25.277063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:25.277573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" released by "process_external" :: held 0.588s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:29:25.278287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:25.278705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:29:25.279257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:25.280014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:29:25.324722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b3aa37a-cd07-44e2-853e-6b665ca869c7]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.376720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d435ccfa-762d-46b3-9a36-fac94b6f4885]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.438951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1aff5c2c-859e-4aa4-be51-4b32788a8ed6]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.489815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dea4d392-d2d7-4782-83cb-c85e11e6a0bb]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.490539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:29:25.491044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:25.492800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:25.493255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" released by "process_address_scope" :: held 0.215s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:29:25.493864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (fc69ce97-e3e0-4d15-8e65-00ab7def92c1), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:25.494505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:25.495292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:29:25.495697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:29:25.496147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:29:25.537724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d969dbb5-8235-458f-985f-5443266df54a]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.634059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b1115c6-cf79-4cb6-a6e3-495ef344b7ac]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.680301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3630d64-3a1f-4dd8-9e96-1dddd64bdad2]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:29:25 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:29:25 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.780905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e10f466-6af5-40b9-99f3-d26f440fe3e2]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.781417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:29:25.781765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:25.782076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:29:25.782366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1" released by "apply_metadata_nat_rules" :: held 0.287s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:29:25.782926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fc69ce97-e3e0-4d15-8e65-00ab7def92c1.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fc69ce97-e3e0-4d15-8e65-00ab7def92c1.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-fc69ce97-e3e0-4d15-8e65-00ab7def92c1 Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/fc69ce97-e3e0-4d15-8e65-00ab7def92c1.pid.haproxy Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID fc69ce97-e3e0-4d15-8e65-00ab7def92c1 Jul 01 09:29:25.783387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:29:25.785802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/fc69ce97-e3e0-4d15-8e65-00ab7def92c1.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:29:25.866329 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-fc69ce97-e3e0-4d15-8e65-00ab7def92c1[129065]: Proxy listener started. Jul 01 09:29:25.871134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router fc69ce97-e3e0-4d15-8e65-00ab7def92c1 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:29:25.871964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for fc69ce97-e3e0-4d15-8e65-00ab7def92c1, update_id dab1f34a-4abd-4e15-a235-3ec080c34a37. Time elapsed: 2.250 Jul 01 09:29:25.872506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fc69ce97-e3e0-4d15-8e65-00ab7def92c1, action 1, priority 1, update_id 127a6418-9b56-47d8-9955-abc13af3520f. Wait time elapsed: 1.550 Jul 01 09:29:25.873316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fc69ce97-e3e0-4d15-8e65-00ab7def92c1, action 1, priority 1, update_id 127a6418-9b56-47d8-9955-abc13af3520f. Wait time elapsed: 1.551 Jul 01 09:29:25.873717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router fc69ce97-e3e0-4d15-8e65-00ab7def92c1 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:29:25.874141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (fc69ce97-e3e0-4d15-8e65-00ab7def92c1), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:25.918780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-27af013b-3408-4151-bb8d-050c4d336ff9 tempest-PortsTestJSON-806010939 tempest-PortsTestJSON-806010939-project] Got routers updated notification :['50ac41ef-2def-4c27-ba1b-8893b0287e65'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:25.919832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3eee444-9004-4ab5-bebf-36a0406a8485]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.922603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 50ac41ef-2def-4c27-ba1b-8893b0287e65, action 3, priority 1, update_id 330688fe-600c-4519-9421-a3720968c139. Wait time elapsed: 0.003 Jul 01 09:29:25.923125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 50ac41ef-2def-4c27-ba1b-8893b0287e65, action 3, priority 1, update_id 330688fe-600c-4519-9421-a3720968c139. Wait time elapsed: 0.004 Jul 01 09:29:25.924075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:25.929417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d440818-358a-41e5-8962-7a23bfb69b50]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.931081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29be3ac9-d1c3-4ee5-93cd-61e013f1a0b7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.932207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router fc69ce97-e3e0-4d15-8e65-00ab7def92c1 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:29:25.933405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c390c886-c0d6-4faa-8dac-7e9f81d082dd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.976190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa15f485-62c8-4265-9605-8121061669bf]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 266071, 'reachable_time': 27836, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 129071, 'error': None, 'target': 'qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:25.976938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:25.977211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:26.005754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93fa8b3f-6b6f-4e3d-b4c4-3c8cb3f49cf4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 266071, 'reachable_time': 27836, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 129072, 'error': None, 'target': 'qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:26.007049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: fc69ce97-e3e0-4d15-8e65-00ab7def92c1 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:29:26.007895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fc69ce97-e3e0-4d15-8e65-00ab7def92c1.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fc69ce97-e3e0-4d15-8e65-00ab7def92c1.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:26.008386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fc69ce97-e3e0-4d15-8e65-00ab7def92c1.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fc69ce97-e3e0-4d15-8e65-00ab7def92c1.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:29:26.008792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for fc69ce97-e3e0-4d15-8e65-00ab7def92c1 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:29:26.009686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router fc69ce97-e3e0-4d15-8e65-00ab7def92c1 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:29:26.010802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[625211d3-c840-4ae1-8430-12e9899a36bb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:26.029980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e23c72fd-f452-4327-97e9-3f76054a5fe0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 266071, 'reachable_time': 27836, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 129073, 'error': None, 'target': 'qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:26.040780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-fc69ce97-e3e0-4d15-8e65-00ab7def92c1 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:29:26.041184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0dcc169-d5fd-40bc-885e-c22e33ec56b4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:26.041621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info fc69ce97-e3e0-4d15-8e65-00ab7def92c1 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:29:26.041965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (fc69ce97-e3e0-4d15-8e65-00ab7def92c1), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:29:26.042308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:26.042586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:26.042883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:26.043147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:26.043489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for fc69ce97-e3e0-4d15-8e65-00ab7def92c1, update_id 127a6418-9b56-47d8-9955-abc13af3520f. Time elapsed: 0.171 Jul 01 09:29:26.314802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.391s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:26.315267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 50ac41ef-2def-4c27-ba1b-8893b0287e65 was not found. Performing router cleanup Jul 01 09:29:26.316502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[297b00a7-753b-48ba-9f4e-e7333c339aaa]: (4, ['qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:26.316997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:26.317268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:26.317526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 50ac41ef-2def-4c27-ba1b-8893b0287e65, update_id 330688fe-600c-4519-9421-a3720968c139. Time elapsed: 0.395 Jul 01 09:29:26.533758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-63b3f9e1-42eb-4680-9edb-9c10a3b3c741 tempest-PortsTestJSON-806010939 tempest-PortsTestJSON-806010939-project] Got router deleted notification for 50ac41ef-2def-4c27-ba1b-8893b0287e65 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:29:26.534710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 50ac41ef-2def-4c27-ba1b-8893b0287e65, action 1, priority 1, update_id f9b0d926-8fa2-45f4-9eed-9854e30a865b. Wait time elapsed: 0.001 Jul 01 09:29:26.535150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 50ac41ef-2def-4c27-ba1b-8893b0287e65, action 1, priority 1, update_id f9b0d926-8fa2-45f4-9eed-9854e30a865b. Wait time elapsed: 0.001 Jul 01 09:29:26.535633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 50ac41ef-2def-4c27-ba1b-8893b0287e65 was not found. Performing router cleanup Jul 01 09:29:26.536955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0389eb50-5149-4f93-bb0e-4c557d836883]: (4, ['qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:26.538078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:26.538524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:26.538944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 50ac41ef-2def-4c27-ba1b-8893b0287e65, update_id f9b0d926-8fa2-45f4-9eed-9854e30a865b. Time elapsed: 0.004 Jul 01 09:29:35.077801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:29:39.357134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-7ce1587c-42c1-4c50-9fc3-16724c66a2b8 tempest-RoutersNegativeIpV6Test-2079169510 tempest-RoutersNegativeIpV6Test-2079169510-project] Got routers updated notification :['528d9dcb-d5c9-4236-81eb-e7c62a015c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:39.359404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 528d9dcb-d5c9-4236-81eb-e7c62a015c91, action 3, priority 1, update_id 4178ed96-a2ab-4541-a2d2-12969bce3acf. Wait time elapsed: 0.001 Jul 01 09:29:39.360706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 528d9dcb-d5c9-4236-81eb-e7c62a015c91, action 3, priority 1, update_id 4178ed96-a2ab-4541-a2d2-12969bce3acf. Wait time elapsed: 0.001 Jul 01 09:29:39.361453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:39.598709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.237s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:39.599146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 528d9dcb-d5c9-4236-81eb-e7c62a015c91 was not found. Performing router cleanup Jul 01 09:29:39.602306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be4f7e6c-0685-4607-969b-ddf777edf7a3]: (4, ['qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:39.603340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:39.603791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:39.604150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 528d9dcb-d5c9-4236-81eb-e7c62a015c91, update_id 4178ed96-a2ab-4541-a2d2-12969bce3acf. Time elapsed: 0.245 Jul 01 09:29:42.559217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-e00fa60b-7fd2-40c0-a8e0-b2f99c68e698 tempest-RoutersNegativeIpV6Test-2079169510 tempest-RoutersNegativeIpV6Test-2079169510-project] Got routers updated notification :['528d9dcb-d5c9-4236-81eb-e7c62a015c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:42.561211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 528d9dcb-d5c9-4236-81eb-e7c62a015c91, action 3, priority 1, update_id 63b881d1-15d4-443d-8053-e3016d00bde2. Wait time elapsed: 0.002 Jul 01 09:29:42.561692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 528d9dcb-d5c9-4236-81eb-e7c62a015c91, action 3, priority 1, update_id 63b881d1-15d4-443d-8053-e3016d00bde2. Wait time elapsed: 0.002 Jul 01 09:29:42.562098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:43.042971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.480s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:43.043692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 528d9dcb-d5c9-4236-81eb-e7c62a015c91 was not found. Performing router cleanup Jul 01 09:29:43.045213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a35b6f29-1d2c-4580-b930-8704a4503003]: (4, ['qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:43.046337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:43.047077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:43.048243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 528d9dcb-d5c9-4236-81eb-e7c62a015c91, update_id 63b881d1-15d4-443d-8053-e3016d00bde2. Time elapsed: 0.487 Jul 01 09:29:47.566287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-91013a1e-e128-4368-9a4b-ade72830e808 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:47.567823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 8753eb3b-2568-4e43-9897-7b406a1371d9. Wait time elapsed: 0.001 Jul 01 09:29:47.570837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 8753eb3b-2568-4e43-9897-7b406a1371d9. Wait time elapsed: 0.001 Jul 01 09:29:47.571159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:47.770203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.199s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:47.770703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:29:47.773482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ac8b1c1-f669-4812-a3c1-52ab15ef22b9]: (4, ['qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:47.774250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:47.774689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:47.775107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id 8753eb3b-2568-4e43-9897-7b406a1371d9. Time elapsed: 0.207 Jul 01 09:29:48.565676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-4a5e5a62-45eb-4cbc-9004-07a64bef1922 tempest-RoutersNegativeIpV6Test-2079169510 tempest-RoutersNegativeIpV6Test-2079169510-project] Got routers updated notification :['528d9dcb-d5c9-4236-81eb-e7c62a015c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:48.567398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 528d9dcb-d5c9-4236-81eb-e7c62a015c91, action 3, priority 1, update_id cdfd970d-acee-44c4-9885-f82e9af8ab62. Wait time elapsed: 0.001 Jul 01 09:29:48.569174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 528d9dcb-d5c9-4236-81eb-e7c62a015c91, action 3, priority 1, update_id cdfd970d-acee-44c4-9885-f82e9af8ab62. Wait time elapsed: 0.002 Jul 01 09:29:48.569639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:48.878821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.309s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:48.879081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 528d9dcb-d5c9-4236-81eb-e7c62a015c91 was not found. Performing router cleanup Jul 01 09:29:48.883292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d4a5efe-e6d3-4654-a2d0-adb3faaa3a3d]: (4, ['qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:48.884397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:48.884671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:48.884931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 528d9dcb-d5c9-4236-81eb-e7c62a015c91, update_id cdfd970d-acee-44c4-9885-f82e9af8ab62. Time elapsed: 0.318 Jul 01 09:29:51.811400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-e465e2da-3cae-4ae0-910e-94d6d40b5fd4 tempest-RoutersNegativeIpV6Test-2079169510 tempest-RoutersNegativeIpV6Test-2079169510-project] Got routers updated notification :['528d9dcb-d5c9-4236-81eb-e7c62a015c91'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:51.813351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 528d9dcb-d5c9-4236-81eb-e7c62a015c91, action 3, priority 1, update_id 6f164c6a-575f-4a6b-a177-a10154daad03. Wait time elapsed: 0.001 Jul 01 09:29:51.815567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 528d9dcb-d5c9-4236-81eb-e7c62a015c91, action 3, priority 1, update_id 6f164c6a-575f-4a6b-a177-a10154daad03. Wait time elapsed: 0.003 Jul 01 09:29:51.816252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:52.017009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.201s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:52.017676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 528d9dcb-d5c9-4236-81eb-e7c62a015c91 was not found. Performing router cleanup Jul 01 09:29:52.019013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d15cee61-e0f7-4cb9-883c-f83b240be916]: (4, ['qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:52.019734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:52.020159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:52.021461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 528d9dcb-d5c9-4236-81eb-e7c62a015c91, update_id 6f164c6a-575f-4a6b-a177-a10154daad03. Time elapsed: 0.207 Jul 01 09:29:52.386779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-bd14c067-2dd1-45eb-86a0-d4eb2382b958 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:52.387833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 574de8d6-ed78-47bf-89ea-35d30122e73d. Wait time elapsed: 0.001 Jul 01 09:29:52.388453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 574de8d6-ed78-47bf-89ea-35d30122e73d. Wait time elapsed: 0.001 Jul 01 09:29:52.389085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:53.048227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.659s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:53.050323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:29:53.052089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81940290-39e1-42da-96f6-d9de95a03d84]: (4, ['qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:53.053525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:53.054019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:53.056740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id 574de8d6-ed78-47bf-89ea-35d30122e73d. Time elapsed: 0.667 Jul 01 09:29:59.257340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-590b19a4-0fc1-4311-94f9-3e2fb67d6410 tempest-RoutersNegativeIpV6Test-2079169510 tempest-RoutersNegativeIpV6Test-2079169510-project] Got router deleted notification for 528d9dcb-d5c9-4236-81eb-e7c62a015c91 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:29:59.258379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 528d9dcb-d5c9-4236-81eb-e7c62a015c91, action 1, priority 1, update_id fea52c40-2209-4d9c-876c-63de581b60d7. Wait time elapsed: 0.001 Jul 01 09:29:59.258754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 528d9dcb-d5c9-4236-81eb-e7c62a015c91, action 1, priority 1, update_id fea52c40-2209-4d9c-876c-63de581b60d7. Wait time elapsed: 0.001 Jul 01 09:29:59.259100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 528d9dcb-d5c9-4236-81eb-e7c62a015c91 was not found. Performing router cleanup Jul 01 09:29:59.260454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d7b3691-88aa-41fa-bb62-8cd19565e490]: (4, ['qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:59.262076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:59.262607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:59.263084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 528d9dcb-d5c9-4236-81eb-e7c62a015c91, update_id fea52c40-2209-4d9c-876c-63de581b60d7. Time elapsed: 0.005 Jul 01 09:29:59.488126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-42542e61-c414-4ab0-813d-48a3fe02ac06 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:29:59.489371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 53569a83-e65e-473a-a1b4-65d78d071224. Wait time elapsed: 0.001 Jul 01 09:29:59.489844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 53569a83-e65e-473a-a1b4-65d78d071224. Wait time elapsed: 0.001 Jul 01 09:29:59.490338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:29:59.784440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.294s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:29:59.784775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:29:59.786211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92a5271a-bcac-42c6-8fa6-0c2d6c230198]: (4, ['qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:29:59.786854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:29:59.787119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:29:59.787463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id 53569a83-e65e-473a-a1b4-65d78d071224. Time elapsed: 0.298 Jul 01 09:30:00.846568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-defb5905-12f6-495a-a4c0-aeb5fa3a8fce tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:00.848882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 031b1a2e-84d4-4490-bb86-b368a4564395. Wait time elapsed: 0.001 Jul 01 09:30:00.849355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 031b1a2e-84d4-4490-bb86-b368a4564395. Wait time elapsed: 0.001 Jul 01 09:30:00.849798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:01.294710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.445s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:01.296291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:30:01.301854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf236666-26b5-4eac-862b-134e13664bb9]: (4, ['qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:01.307517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:01.308381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:01.309719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id 031b1a2e-84d4-4490-bb86-b368a4564395. Time elapsed: 0.461 Jul 01 09:30:02.915351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8091a3c6-220e-4d3d-9638-4924b609084b tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['d7ad1699-9b3a-4784-abcc-b862984f240b'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:02.917352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update d7ad1699-9b3a-4784-abcc-b862984f240b, action 3, priority 1, update_id 0debb92d-7dfc-4133-87cb-066d114c550d. Wait time elapsed: 0.001 Jul 01 09:30:02.917606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for d7ad1699-9b3a-4784-abcc-b862984f240b, action 3, priority 1, update_id 0debb92d-7dfc-4133-87cb-066d114c550d. Wait time elapsed: 0.002 Jul 01 09:30:02.917964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:03.289597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.371s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:03.289861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router d7ad1699-9b3a-4784-abcc-b862984f240b was not found. Performing router cleanup Jul 01 09:30:03.292272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9dbf116-570e-44d5-903b-bd723c47d40e]: (4, ['qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:03.292788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:03.293285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:03.293673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for d7ad1699-9b3a-4784-abcc-b862984f240b, update_id 0debb92d-7dfc-4133-87cb-066d114c550d. Time elapsed: 0.377 Jul 01 09:30:07.880928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-3e40bd78-fc53-455d-8be5-b2349c4d1629 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:07.885310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id b5166703-ffdb-450b-aeba-7b98939e8f38. Wait time elapsed: 0.001 Jul 01 09:30:07.885812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id b5166703-ffdb-450b-aeba-7b98939e8f38. Wait time elapsed: 0.003 Jul 01 09:30:07.887407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:08.162089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-31e79e3f-a286-44cd-9118-5595a0c324ba tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['d7ad1699-9b3a-4784-abcc-b862984f240b'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:08.163449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update d7ad1699-9b3a-4784-abcc-b862984f240b, action 3, priority 1, update_id dbdd02ef-d050-4bc3-912b-0d69cb65b47b. Wait time elapsed: 0.001 Jul 01 09:30:08.163963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for d7ad1699-9b3a-4784-abcc-b862984f240b, action 3, priority 1, update_id dbdd02ef-d050-4bc3-912b-0d69cb65b47b. Wait time elapsed: 0.001 Jul 01 09:30:08.164564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:08.455281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.291s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:08.455846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router d7ad1699-9b3a-4784-abcc-b862984f240b was not found. Performing router cleanup Jul 01 09:30:08.458117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10b39d17-49b5-4b40-a8f2-9a1fc9e92418]: (4, ['qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:08.459568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:08.460323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:08.461065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for d7ad1699-9b3a-4784-abcc-b862984f240b, update_id dbdd02ef-d050-4bc3-912b-0d69cb65b47b. Time elapsed: 0.297 Jul 01 09:30:08.511323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.625s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:08.513181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:30:08.520634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41a5dd54-8661-40b0-a40f-22f8fb25d53f]: (4, ['qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:08.521688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:08.526605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.005s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:08.527596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id b5166703-ffdb-450b-aeba-7b98939e8f38. Time elapsed: 0.644 Jul 01 09:30:09.474758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-2003d2e6-d4e0-40ab-9f17-efb4ad9306ba tempest-TestNetworkAdvancedServerOps-636282898 tempest-TestNetworkAdvancedServerOps-636282898-project] Got routers updated notification :['58317645-6e22-4a4b-8851-15323d1dba25'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:09.477196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 1, update_id 5047d151-2cc9-4333-ae13-1bfa2875eddf. Wait time elapsed: 0.001 Jul 01 09:30:09.477869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 1, update_id 5047d151-2cc9-4333-ae13-1bfa2875eddf. Wait time elapsed: 0.001 Jul 01 09:30:09.478400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:10.540507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-fdc2155e-935b-4fc5-bdbf-d4901c262e57 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:10.542174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id f5331572-f268-462b-8072-fde9e5a8534b. Wait time elapsed: 0.001 Jul 01 09:30:10.542845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id f5331572-f268-462b-8072-fde9e5a8534b. Wait time elapsed: 0.001 Jul 01 09:30:10.543832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:10.703226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.159s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:10.703513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:30:10.705158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a60d77f-be30-4269-ae73-dfa89d300efd]: (4, ['qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:10.705669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:10.705943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:10.706193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id f5331572-f268-462b-8072-fde9e5a8534b. Time elapsed: 0.164 Jul 01 09:30:10.959985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-fdc2155e-935b-4fc5-bdbf-d4901c262e57 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:10.961466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id c0aaeb29-d369-4c53-8fd3-fa776f07e6fd. Wait time elapsed: 0.001 Jul 01 09:30:10.961994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id c0aaeb29-d369-4c53-8fd3-fa776f07e6fd. Wait time elapsed: 0.001 Jul 01 09:30:10.962902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:11.457222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.493s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:11.457476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:30:11.459106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[984ce11c-6ef7-4051-b312-95b71e2a3896]: (4, ['qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:11.459933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:11.460544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:11.460957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id c0aaeb29-d369-4c53-8fd3-fa776f07e6fd. Time elapsed: 0.500 Jul 01 09:30:11.654221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-bad8768c-5ca5-4d64-a822-d8a1df04ed73 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:11.656714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id f1445039-8a07-4c51-b678-e88cc37262ad. Wait time elapsed: 0.002 Jul 01 09:30:11.659639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id f1445039-8a07-4c51-b678-e88cc37262ad. Wait time elapsed: 0.002 Jul 01 09:30:11.659639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:11.781126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-f202fcd8-9e1c-428f-bce1-1924c127a44d tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['d7ad1699-9b3a-4784-abcc-b862984f240b'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:11.782475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update d7ad1699-9b3a-4784-abcc-b862984f240b, action 3, priority 1, update_id 160ceb40-c83c-40e8-875a-69dc32c43529. Wait time elapsed: 0.001 Jul 01 09:30:11.782900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for d7ad1699-9b3a-4784-abcc-b862984f240b, action 3, priority 1, update_id 160ceb40-c83c-40e8-875a-69dc32c43529. Wait time elapsed: 0.001 Jul 01 09:30:11.783729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:11.794516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.136s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:11.794778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:30:11.796763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c08d2028-e2e6-4cfa-a6bd-7ab9524ddac7]: (4, ['qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:11.806142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:11.806674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.009s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:11.807098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id f1445039-8a07-4c51-b678-e88cc37262ad. Time elapsed: 0.150 Jul 01 09:30:12.107566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.324s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:12.109041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router d7ad1699-9b3a-4784-abcc-b862984f240b was not found. Performing router cleanup Jul 01 09:30:12.113875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c48f26d2-ab57-4902-81cc-ab485a3ed254]: (4, ['qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:12.114828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:12.115928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:12.116353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for d7ad1699-9b3a-4784-abcc-b862984f240b, update_id 160ceb40-c83c-40e8-875a-69dc32c43529. Time elapsed: 0.334 Jul 01 09:30:12.162686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-469c19e1-ba78-403b-92d5-d39ec50efd3e tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['e1dc75ae-b350-4801-91dc-89215c41ca50'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:12.164837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 1, update_id d37615a9-34b6-4d4d-b579-97954c960dd9. Wait time elapsed: 0.001 Jul 01 09:30:12.165312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 1, update_id d37615a9-34b6-4d4d-b579-97954c960dd9. Wait time elapsed: 0.002 Jul 01 09:30:12.165875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:13.979487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.482s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:13.980571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:13.988459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:13.990835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:13.993412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:13.999257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb664157-222e-4392-9b46-0eef2a8ddf82]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.072532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9995847a-f6d8-4141-b226-b5ce9f3b9eea]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.145203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bacda7a-68cb-4ebf-a288-e4e156d8ee5b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.225980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00b0a7c3-3c7f-449b-97cf-5256b20f2c85]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 130551, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.259847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac9c6fcc-34c2-4432-bafe-68e8e2000a8e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.294620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f380482d-d96c-4445-99f3-4d973c3afbd5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.328662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c62fdf60-6e10-4fd9-8916-0814265b886d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.383302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7363c8e6-b8aa-465a-a37f-279b29941a7f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.384067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:14.385774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:14.386066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:14.440461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c92660bc-46ce-4616-8ab4-771a40e836e0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 271048, 'reachable_time': 18200, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 130565, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.441284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:14.441549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:14.441979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:30:14.442234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:30:14.442563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:14.442818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:14.443397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '6abcecf7-8aba-4321-83e2-43a0ac8d9c3a', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:48:1c:04', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '58317645-6e22-4a4b-8851-15323d1dba25', 'device_owner': 'network:router_gateway', 'standard_attr_id': 1283, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.248', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:30:07Z', 'updated_at': '2022-07-01T09:30:12Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-6abcecf7-8a), ns(qrouter-58317645-6e22-4a4b-8851-15323d1dba25) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:30:14.476358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:14.476683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0ccbcb6-1450-4f10-bd6d-d9269d75a18a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.480325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82bb93cc-9483-48fa-b0d6-b70fcb50b7d4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.481489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-6abcecf7-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:14.482036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:30:14.483030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-6abcecf7-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:14.484184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-6abcecf7-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:14.484866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-6abcecf7-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:14.485412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-6abcecf7-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:14.486006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-6abcecf7-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6abcecf7-8aba-4321-83e2-43a0ac8d9c3a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:48:1c:04'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:14.523056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.357s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:14.524333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:14.530256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:14.530923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:14.531436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:14.538189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b28f39bf-8a07-4c56-8ef7-597cce8cc110]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.538834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2372260-2ace-4082-b0ff-b4769791157b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.573432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[446c22e9-fac9-4bb0-a360-8e98b52a5ac5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.591318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5cbd02dd-2b4b-49bc-ba6b-b53871c12ecb]: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:14.591318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:14.591318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:14.591318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:14.591318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:14.591318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:14.591318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:14.591318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:14.591318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:14.591318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:14.591318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:14.591318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:14.591318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:14.592570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cbd02dd-2b4b-49bc-ba6b-b53871c12ecb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.592570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:30:14.593411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:30:14.615143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a483386-94cb-40be-b7ae-705d7a7daab0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.630614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[598901d9-955a-47a7-8991-5ab00fa8ddb7]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.675483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13178fc0-0f4c-4528-84c3-ec25296a1559]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 130588, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.716394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed5d0a53-76b3-4bb1-bdf0-319ef99e59ee]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.737130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[975b07a9-e1d0-4677-ad71-e627dcbde0f8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.765214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a795e839-0043-4fb7-b84a-5f9f46a059e4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.794825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4fd5354-7e0f-4d39-ab33-4a8f82313d64]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.795630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:14.797919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:14.798258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:14.835814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9acd7bcb-557f-4d7d-a74e-5ea2c3c70ef8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 271093, 'reachable_time': 40424, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 130605, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.836792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:14.837291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:14.838419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:30:14.838847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:30:14.839342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:14.839857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:14.840396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '105dbb76-6459-4ce9-a126-c111de1babfc', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:9e:72:fc', 'admin_state_up': True, 'status': 'DOWN', 'device_id': 'e1dc75ae-b350-4801-91dc-89215c41ca50', 'device_owner': 'network:router_gateway', 'standard_attr_id': 1284, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.160', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:30:08Z', 'updated_at': '2022-07-01T09:30:13Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-105dbb76-64), ns(qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:30:14.854157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8a09fc1-7ef0-43c0-866c-8875a410c0a1]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.870588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:14.871237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[203608c2-dcbf-42c8-bb6f-e75da82cefd9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.875255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0508954b-e7b6-4d78-91dc-4abaebec47b5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.876517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-105dbb76-64, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:14.877016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:30:14.877930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-105dbb76-64, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:14.878810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-105dbb76-64, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:14.879408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-105dbb76-64, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:14.879793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-105dbb76-64, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:14.880176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-105dbb76-64, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '105dbb76-6459-4ce9-a126-c111de1babfc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:72:fc'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:14.892717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a04a915b-f3bb-4412-af65-0084f0a96710]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.931088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d240f26-c007-46dc-88d9-bee851b7f168]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.987826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[090164de-80b4-4bf4-b1df-d924acd5769e]: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:14.987826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:14.987826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:14.987826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:14.987826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:14.987826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:14.987826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:14.987826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:14.987826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:14.987826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:14.987826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:14.987826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:14.987826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:14.989643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[090164de-80b4-4bf4-b1df-d924acd5769e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:14.989643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:30:14.989643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:30:15.015816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a44bb17b-e124-4d2d-b7c9-d0b13535cfd0]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.074493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:15.075117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:15.075513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:15.075810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:15.081369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:30:15.105439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f18ad717-e392-4154-90a3-51750828d260]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.106671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:30:15.107314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:30:15.108928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:30:15.109609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" released by "process_external" :: held 0.667s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:15.110874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:15.115266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:15.120090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[777c8fd9-e155-4784-b4a7-7c2982460fb5]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.121153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:15.121824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:15.122245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:15.123257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:15.126555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93d238a1-c393-4449-9427-b65b71231dbb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.131004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[972ba9ef-0e9b-40d6-8705-ae28381e3e6c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.132270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7574f2cc-4488-4b2a-9784-19cd3bb6fd8b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.176171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d0f389f-ca5b-42a3-9d60-19f6297c2b88]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 271048, 'reachable_time': 18200, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 130633, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.194012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:15.194678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c74705ca-28a3-4855-ab98-37f6a564fb0d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:15.198087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:07Z", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:12Z", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:30:15.211467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:15.243227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 5047d151-2cc9-4333-ae13-1bfa2875eddf. Wait time elapsed: 5.747 Jul 01 09:30:15.243227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 5047d151-2cc9-4333-ae13-1bfa2875eddf. Wait time elapsed: 5.760 Jul 01 09:30:15.243227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:15.452072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55177b8c-f034-48ad-aa31-9b68862e6ef7]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.494445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52c3e4cb-5917-458a-b170-1e4827fcdf27]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.880462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3510f3ea-1b7d-4d0e-9b2a-0cf2eda8964d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.882370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:30:15.883524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:30:15.884119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:30:15.884565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" released by "process_external" :: held 1.046s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:15.885632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:15.890566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:15.899849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75f65a6f-9b6f-47a1-af86-7a0d86e78665]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.900863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:15.901390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:15.901841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:15.902414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:15.904236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[457f9aba-f978-4d1c-9b75-66f5047f3b6d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.910608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[149704a0-fb04-4785-8562-513c4222ac78]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.912596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a6e9fbe-1246-4735-9d4c-bd56d36f00d7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.950231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf6e7479-975f-4190-96f5-766a72f64728]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 271093, 'reachable_time': 40424, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 130664, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.960227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:15.960512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4266f63e-189c-4e1c-9804-85ef15e6153b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:15.961825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:09Z", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:13Z", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id d37615a9-34b6-4d4d-b579-97954c960dd9. Wait time elapsed: 3.800 Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id d37615a9-34b6-4d4d-b579-97954c960dd9. Wait time elapsed: 3.800 Jul 01 09:30:15.970475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:16.685137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c79d1744-dff1-4c16-b119-ad545bc4af3f tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['d7ad1699-9b3a-4784-abcc-b862984f240b'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:16.688932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update d7ad1699-9b3a-4784-abcc-b862984f240b, action 3, priority 1, update_id 10b28af5-296f-4ce6-ae64-21e3467205ee. Wait time elapsed: 0.003 Jul 01 09:30:16.689789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for d7ad1699-9b3a-4784-abcc-b862984f240b, action 3, priority 1, update_id 10b28af5-296f-4ce6-ae64-21e3467205ee. Wait time elapsed: 0.004 Jul 01 09:30:16.690219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:17.209953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.246s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:17.210826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:17.212009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:17.212643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:17.213162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:17.214607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f3d5afa-f5c6-448e-b488-a69c643920af]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.236678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15aed2b6-87ba-48a2-88d0-292610ac15d1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.266153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a44de91c-5755-4287-9474-6c13bf90493f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.314876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-d44bde2c-84b9-4744-9760-5d815a79acc4 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['e1dc75ae-b350-4801-91dc-89215c41ca50'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:17.324435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87201772-aa88-45f1-a61b-57f809086e34]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 130723, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.351859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a0be738-abfd-4631-befb-7b635ff99592]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.372882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.683s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:17.373134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router d7ad1699-9b3a-4784-abcc-b862984f240b was not found. Performing router cleanup Jul 01 09:30:17.375973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[321e7f0d-3cd2-4167-a5c1-2c1cf8f7e064]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.376669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:17.377514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:17.377854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for d7ad1699-9b3a-4784-abcc-b862984f240b, update_id 10b28af5-296f-4ce6-ae64-21e3467205ee. Time elapsed: 0.689 Jul 01 09:30:17.386743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df9ceec2-dd9c-4ee6-8f3a-2e4745b0c071]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.449112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa8099eb-b61c-4c72-943c-eedd7cc0e9c6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.493602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2ba2d06-127f-408a-ae64-9eed9fb98289]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.494545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:17.495034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:17.495636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:17.496138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:17.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:17.526121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1770372-116d-4f0c-aed0-03f16b3a8621]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.531444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5553ff8-9b32-4a98-9ce9-45c52cfb1798]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.532684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:17.533252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:30:17.534216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:17.535233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:17.536751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:17.537377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:17.537987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:17.601885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a4d3559-9eab-4f44-9e23-113424edb4d3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.628785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-07d1542c-17e1-4f59-b930-54289241d933 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got router deleted notification for d7ad1699-9b3a-4784-abcc-b862984f240b {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:30:17.629794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update d7ad1699-9b3a-4784-abcc-b862984f240b, action 1, priority 1, update_id 88ca31d9-9b0c-4171-8658-e2f3c284866f. Wait time elapsed: 0.001 Jul 01 09:30:17.630203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for d7ad1699-9b3a-4784-abcc-b862984f240b, action 1, priority 1, update_id 88ca31d9-9b0c-4171-8658-e2f3c284866f. Wait time elapsed: 0.001 Jul 01 09:30:17.630457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router d7ad1699-9b3a-4784-abcc-b862984f240b was not found. Performing router cleanup Jul 01 09:30:17.631670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd60a249-b9f7-4fa6-90d2-50c9efb34daf]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.632955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:17.633514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:17.633981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for d7ad1699-9b3a-4784-abcc-b862984f240b, update_id 88ca31d9-9b0c-4171-8658-e2f3c284866f. Time elapsed: 0.004 Jul 01 09:30:17.677951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[add4b11d-a520-4e6c-8bb0-8e95ec85583a]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:17.677951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:17.677951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:17.677951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:17.677951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:17.677951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:17.677951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:17.677951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:17.677951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:17.677951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:17.677951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:17.677951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:17.677951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:17.680165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[add4b11d-a520-4e6c-8bb0-8e95ec85583a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:17.682164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:17.685703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:17.692822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3860a408-93d9-450e-9201-dac703bf3f2e]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.694185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:17.695594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:17.696041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:17.696578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:17.698628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8914fdf8-4abc-4389-b31f-a2353612b697]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.699892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74821754-594b-41b0-bf63-d86ebc7f6639]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.703925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fbfd36d-5974-4ba8-aee4-4e95ff2c15e5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.730931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2bff673-15a0-4a16-a7db-ec11942cbdba]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 271358, 'reachable_time': 21476, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 130757, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.735283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:17.736425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b732ed37-a2fb-4a86-adf3-addfd144322b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:17.738078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:13Z", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:15Z", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:17.742920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:17.760906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 1, update_id b7af566d-370a-4752-a709-3059693f78e4. Wait time elapsed: 0.440 Jul 01 09:30:17.760906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 1, update_id b7af566d-370a-4752-a709-3059693f78e4. Wait time elapsed: 0.440 Jul 01 09:30:17.760906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:19.579756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.341s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:19.580378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:19.581115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:19.585678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.004s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:19.586101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:19.596123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[542d2baa-3d97-4d5d-88ff-b36d29de4b13]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:19.676736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6963c3e5-f4f3-4253-b610-3e15665b16d9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:19.724971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[518177dc-ac52-4743-9fa6-c6b857c884f4]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:19.817622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8876fe9-8567-4f20-8aca-83496906173b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 130885, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:19.884309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[018cf022-ae20-4c9d-b854-86fbc87bff7f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:19.920773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[075355de-9efd-4058-8c4a-b33a5bf2d15a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:19.976717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[096adaa4-0c93-4fc6-bf83-3ebea42b66a4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.016902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1ab6a41-130b-44e6-ae80-998e9470fb28]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.019426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:20.020019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:20.020569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:20.021093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:20.023136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-36e1ee82-e6d1-44f9-9cc7-30420f7ab4bd tempest-TestNetworkAdvancedServerOps-636282898 tempest-TestNetworkAdvancedServerOps-636282898-project] Got routers updated notification :['58317645-6e22-4a4b-8851-15323d1dba25'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:20.071472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:20.072076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1248316-0da9-4f6e-ac1f-2b7251e93462]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.078665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87216bfc-c773-44df-8292-250f55361e36]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.080341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:20.080771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:30:20.081951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:20.083211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:20.083957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:20.084592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:20.085207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:20.094659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.338s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:20.094991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:20.095916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:20.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:20.096845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:20.116080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81e3faae-b08c-4d83-9b57-5a177d5ffb69]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.117892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a79f8b7-b7a3-41ac-9a8f-bf9f179438ad]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.169806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[730b1b45-2549-4b2d-a4c4-5aadd7a193be]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.210754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d38080b5-856c-4a3e-9963-2bddf1ae0398]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:20.210754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:20.210754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:20.210754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:20.210754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:20.210754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:20.210754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:20.210754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:20.210754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:20.210754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:20.210754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:20.210754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:20.210754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:20.212430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d38080b5-856c-4a3e-9963-2bddf1ae0398]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:20.213549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:20.217233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:20.223668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d88354dc-3d07-4ec0-9cd8-87c45a27b091]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.224598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06656e9b-ad76-4b3f-9c72-7dab791cae26]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.227232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:20.227848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:20.228272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:20.228862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:20.230193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e50a4415-77d1-4d3e-9101-45684eeb7a5e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.231974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2bbe6d05-dd37-45b2-aa7c-845fd1d6379e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.233874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85ec3c6f-762e-4eb2-b994-16e2ba2d5ac4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.270682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d28d2a51-933c-458c-8fdc-a6c36e1502cb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 271608, 'reachable_time': 16184, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 130938, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.274626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:20.275929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2107fd85-727d-4a30-904a-7b45c50c92b9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:20.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:12Z", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:15Z", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:20.294557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:20.310127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 1, update_id 7b94701f-1773-4678-a963-a5ac13db9ced. Wait time elapsed: 0.283 Jul 01 09:30:20.310127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 1, update_id 7b94701f-1773-4678-a963-a5ac13db9ced. Wait time elapsed: 0.283 Jul 01 09:30:20.310127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:20.317214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e04b741-7623-4248-8f28-598b300405d6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 130937, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.379149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41f80088-f4bf-4853-bd0c-71373b72c719]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.445166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3cec106b-10cc-49d8-9933-0421ffb0cc8c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.491871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98c42879-1de0-4049-98ce-3f07f1a5735c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.531387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7a96158-2718-4643-8a30-b34bb31d9e18]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.533157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:20.533695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:20.534196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:20.534743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:20.565450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:20.566109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af6b81af-4793-4a92-a91e-5dbbf6ec7502]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.575244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d85ffe9a-bd6c-4466-b73a-7a8605294e5c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.579176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:20.634437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:20.637318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:20.638091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:20.638899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:20.640971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:20.708018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af78eb42-a267-43b3-8467-99897e5bedf7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.743614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c1fb7db8-272a-4fe4-b910-c163b6ba3183]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:20.743614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:20.743614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:20.743614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:20.743614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:20.743614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:20.743614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:20.743614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:20.743614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:20.743614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:20.743614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:20.743614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:20.743614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:20.744847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1fb7db8-272a-4fe4-b910-c163b6ba3183]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:20.746841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:20.751136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:20.756713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5f91ca1-bec8-46d4-bcc7-d982ff5c3e96]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.757709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:20.758221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:20.758476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:20.758874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:20.760021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d72d317-4283-4a3d-abce-9a8ee72634b9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.761371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d807112-f429-45b1-8476-aa194401112e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.763570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3838d85b-be51-4026-922c-8b069f8c60a7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.805341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ddc8123-629e-469b-9040-84347c1eb52f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 271657, 'reachable_time': 18668, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 130976, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63ef8235-2ffa-44ca-b73e-c03f1a72286b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:20.814753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:19Z", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:20.819609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id d37615a9-34b6-4d4d-b579-97954c960dd9. Wait time elapsed: 8.654 Jul 01 09:30:20.841149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id d37615a9-34b6-4d4d-b579-97954c960dd9. Wait time elapsed: 8.654 Jul 01 09:30:20.841149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:21.539473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ca504aa4-2747-4504-9bd7-a8c48d152459 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:21.540632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id dc6ee467-a6ad-45cc-9e67-63ccd6107f2c. Wait time elapsed: 0.001 Jul 01 09:30:21.540888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id dc6ee467-a6ad-45cc-9e67-63ccd6107f2c. Wait time elapsed: 0.001 Jul 01 09:30:21.541183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:21.866277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.048s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:21.867506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:21.868870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:21.869864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:21.870623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:21.872487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e8e0cd3-c08d-47b2-9552-7d1e0a3e458e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:21.896131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68fedad4-f98b-43ac-a2e4-5286560eb9a4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:21.937243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ff8893f-e5b1-42a5-85e1-810dcc184119]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.002516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a9a1c1b-48ca-47d0-bd91-988f96f89abe]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 131023, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.042497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75811bf6-29e9-4e35-80f5-5ecf331ebcb1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.091640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4375fd64-205e-4a33-8016-e151586bee63]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.133491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb75734b-f9e2-4f58-b896-cbcf471a38ca]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.171334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78b28f4f-b958-4336-8052-c2dcc31a3d71]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.172056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:22.172499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:22.172717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:22.173286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:22.208817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:22.209167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43ddae22-1d1a-48ad-b4a8-b44619a2de0c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.217288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3123d224-7b8f-4f94-ae95-3a95156744b0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.219555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:22.255794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:22.258910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:22.260578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:22.262998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:22.263511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:22.323731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a774ea3-4fb4-4d6a-b9d5-3aa310b3fb7f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.368533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.827s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:22.368799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:30:22.370822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed7a274d-765a-4711-86fd-efb2bdecb551]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.371330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:22.371673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:22.371930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id dc6ee467-a6ad-45cc-9e67-63ccd6107f2c. Time elapsed: 0.831 Jul 01 09:30:22.396340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8ae8bbab-a806-4ae1-a7dd-317f04577cea]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:22.396340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:22.396340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:22.396340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:22.396340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:22.396340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:22.396340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:22.396340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:22.396340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:22.396340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:22.396340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:22.396340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:22.396340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:22.397465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ae8bbab-a806-4ae1-a7dd-317f04577cea]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:22.404131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:22.406373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:22.408788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce2dcdde-eb19-4186-923f-6f7227d2c680]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.408788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:22.408788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:22.408788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:22.408788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:22.413246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63b3bce5-f075-4d23-a380-a93312d15c79]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.415203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08dad00d-852e-42bd-b91d-5164bbe52d04]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.417286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a69bcbf6-96b9-4c78-a1cc-3562ed4e4a3c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.460469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50a84065-baf5-430c-b10d-f96d70c99a7b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 271826, 'reachable_time': 32996, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 131064, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.470230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:22.470495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d04308b0-33bb-4ba3-ac83-a3f9e4d6d1b7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:22.471695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:19Z", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:22.476574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:22.498102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b7af566d-370a-4752-a709-3059693f78e4. Wait time elapsed: 5.171 Jul 01 09:30:22.498102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b7af566d-370a-4752-a709-3059693f78e4. Wait time elapsed: 5.171 Jul 01 09:30:22.498102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:23.884528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.397s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:23.885158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:23.885706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:23.886027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:23.886306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:23.887439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87f8021f-41e1-4f7d-93dd-483d627bbe43]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:23.924517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2aa87088-396f-4245-a3a2-d03c00e99b02]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:23.968903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[453798ff-efa7-4101-84fb-c6383eef1f1f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.031201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[012f920f-0a23-4d32-8005-a15e8f532b24]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 131128, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.067469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e896c490-f993-4fb4-8b52-798818e72c54]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.117373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84a7ae4c-1779-49ee-80d9-e490e3c10e9f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.159678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f29df29-93b9-4407-a091-64ffafc5d0eb]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.208977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85926217-efcb-41db-ae2f-b1125cfcb943]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.209568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:24.209844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:24.210174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:24.210449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:24.253742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-72138fd4-d76e-48ba-a82a-f09b041682fd tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:24.254676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 874fcf96-676f-4d03-9042-1b440550db4c. Wait time elapsed: 0.001 Jul 01 09:30:24.255132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 874fcf96-676f-4d03-9042-1b440550db4c. Wait time elapsed: 0.001 Jul 01 09:30:24.255826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:24.264776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:24.265841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5956ded1-be69-4b34-b880-fdb933b73855]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.268847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99da1471-4e49-478e-aa5a-072722ce0274]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.269909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:24.306018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:24.307036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:24.307562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:24.308778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:24.309434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:24.361722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[720e80c0-4bf5-40d3-b697-eb399a70f2d0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d8865c21-13ca-4e06-867e-69721cb55ef1]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:24.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:24.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:24.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:24.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:24.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:24.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:24.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:24.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:24.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:24.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:24.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:24.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:24.405717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8865c21-13ca-4e06-867e-69721cb55ef1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:24.408784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:24.411021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:24.414376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[136e71e7-9c1d-4d8d-9bb4-fc0bf2bcb86a]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.415088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:24.415944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:24.416372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:24.416873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:24.418102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e044617-a89a-4519-9cef-ef2d94455e50]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.419326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05f43065-82c9-4f76-989f-5ed4c126ea6e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.420600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[023ad399-b424-447e-b9aa-78da39ea12f3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.462237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aeeac162-0a43-44a4-8de7-5650e4c995be]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 272029, 'reachable_time': 35168, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 131163, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f30ed2d-e9cb-460a-9a51-40d0a55dec39]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:24.469515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:19Z", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id d37615a9-34b6-4d4d-b579-97954c960dd9. Wait time elapsed: 12.303 Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id d37615a9-34b6-4d4d-b579-97954c960dd9. Wait time elapsed: 12.303 Jul 01 09:30:24.477313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:25.439454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.172s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:25.440854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:30:25.442291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f1d32f0-17dc-441a-9461-abf0e6ee9158]: (4, ['qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:25.446702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:25.447171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:25.447613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id 874fcf96-676f-4d03-9042-1b440550db4c. Time elapsed: 1.193 Jul 01 09:30:25.610223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.143s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:25.610677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:25.612356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:25.612865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:25.613305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:25.616628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[301e55fd-7c61-4cc0-a30d-3ef3448dcb93]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:25.644249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8255755c-2871-44a6-8b05-96dc292caeba]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:25.676949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c0fee0b-51f5-4161-8006-9fd6af8f7e28]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:25.717424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02ad0609-95a9-49aa-aedf-db7abdd4fbe4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 131212, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:25.739900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ef69264-f540-4396-97a0-5ba50d163b07]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:25.781264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a31e60ed-6ea7-4729-8fd0-10299319bfd4]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:25.824223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef60f4a6-08d5-4bfb-98b0-c926445d1568]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:25.858416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30ef400a-be10-4be9-8299-7fa709b21698]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:25.859277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:25.860365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:25.860943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:25.861593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:25.870422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-9faaf628-da16-4f8b-b601-f18adb851a2a tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:25.871583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 60f36e56-461b-4da4-b0e9-d2781b36b9ab. Wait time elapsed: 0.001 Jul 01 09:30:25.872152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 60f36e56-461b-4da4-b0e9-d2781b36b9ab. Wait time elapsed: 0.001 Jul 01 09:30:25.872606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:25.885556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:25.886071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94d6fdd3-aaab-46ec-8d91-bc81510c86e5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:25.895794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b646fe92-9450-4d43-ba95-4db36dcbfd19]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:25.898911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:25.957467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:25.959571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:25.960170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:25.960737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:25.961294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:26.001095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[723c637f-416b-46ba-8a18-9dbf500fd34d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:26.036824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[606feb7d-8658-4c48-a429-a744b3b74572]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:26.036824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:26.036824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:26.036824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:26.036824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:26.036824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:26.036824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:26.036824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:26.036824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:26.036824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:26.036824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:26.036824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:26.036824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:26.037982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[606feb7d-8658-4c48-a429-a744b3b74572]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:26.038084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:26.040013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:26.045836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f45f2603-5242-47db-82cf-a43919542014]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:26.045836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:26.045836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:26.045836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:26.045836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:26.047754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4bd4a9d-10ef-4199-bc07-414f429cdaad]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:26.048976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e85d5fa-8892-460a-8fd1-5548ea085397]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:26.049913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a37f06d1-0109-4d13-bf25-6f7da08aa1a8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:26.070211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b20e0b91-f65c-46e8-8f7d-560123bab09f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 272198, 'reachable_time': 44080, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 131247, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:26.080720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:26.081053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20d8f823-528e-4f68-a37c-15f2567fe177]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:26.082108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:19Z", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:26.085561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b7af566d-370a-4752-a709-3059693f78e4. Wait time elapsed: 8.768 Jul 01 09:30:26.105381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b7af566d-370a-4752-a709-3059693f78e4. Wait time elapsed: 8.768 Jul 01 09:30:26.105381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:26.783202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.910s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:26.783856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:30:26.784982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dffc19e8-14b0-415a-ba26-0517ece497b4]: (4, ['qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:26.785776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:26.786223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:26.786957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id 60f36e56-461b-4da4-b0e9-d2781b36b9ab. Time elapsed: 0.915 Jul 01 09:30:27.424161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.338s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:27.430083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:27.433230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:27.434378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:27.434941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:27.441527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7c4bf41-bd85-4860-9fc4-320ca365a1eb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.468319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[645deb55-0f1d-4500-b573-c0e1656e61f7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.516149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0062661d-e454-4bf6-a906-ce5d54848f8b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.580183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53e38798-5b3c-46fe-9050-beff2c10c3ca]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 131356, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.623652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[346d3ffb-d83d-4ff1-b846-cfd97cc940f3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.655348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b3d52ae-bd12-417a-aa82-5e0ea978faeb]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.681926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31cb49a1-4337-4e44-829e-e524e04625b9]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.732159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb9de8ad-4068-4ea1-9516-9f6070f189cb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.733197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:27.733970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:27.734407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:27.735039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:27.776757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:27.777000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0170b6ab-c769-4f0a-844b-f55fc4969f0f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.780421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1274d0d9-95a4-4992-8d70-bd5a2011a970]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.781790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:27.822055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:27.823488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:27.824111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:27.824666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:27.826059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:27.876647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4b7f3ff-a020-4b9f-bd7c-52d40a3a9ffe]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.919483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[908d3bfd-f5d7-4919-a48b-b8c7d7e0f20f]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:27.919483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:27.919483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:27.919483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:27.919483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:27.919483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:27.919483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:27.919483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:27.919483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:27.919483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:27.919483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:27.919483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:27.919483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:27.920839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[908d3bfd-f5d7-4919-a48b-b8c7d7e0f20f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:27.921345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:27.924369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:27.928402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50690676-4c62-422b-8cba-f88b39eb8235]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.929300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:27.929772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:27.930179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:27.930907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:27.933254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b9640d4-ea8e-44dd-bc32-467a9e93ef88]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.934460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2df054c-7608-46a9-9bea-bec11c00be99]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.935645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6cd0e71b-7698-492f-8ac0-bf11a4dc5c03]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.958616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68106e63-1846-4349-8e65-a74d65bc6364]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 272383, 'reachable_time': 31492, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 131387, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.962961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:27.963448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[699bd885-eee6-4021-a4d8-282b262210bc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:27.966926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:19Z", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:27.970689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:27.987607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id d37615a9-34b6-4d4d-b579-97954c960dd9. Wait time elapsed: 15.808 Jul 01 09:30:27.987607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id d37615a9-34b6-4d4d-b579-97954c960dd9. Wait time elapsed: 15.809 Jul 01 09:30:27.987607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:31.358465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-d499801c-94df-4c89-9ae8-fefa90a48557 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:31.362512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 618a1ce3-bebf-4f7d-8068-d5919c92e9fc. Wait time elapsed: 0.002 Jul 01 09:30:31.362987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 618a1ce3-bebf-4f7d-8068-d5919c92e9fc. Wait time elapsed: 0.004 Jul 01 09:30:31.370087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:31.496615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 11.189s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:31.496901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:31.497820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:31.498485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:31.499044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:31.500558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a799cad8-6e4a-4423-8a9c-c637ea8ae627]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.523308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.158s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:31.523844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:30:31.527955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1319ad2-7f40-4fc6-9d04-865b45c07d72]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.532824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:31.533306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:31.535722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id 618a1ce3-bebf-4f7d-8068-d5919c92e9fc. Time elapsed: 0.175 Jul 01 09:30:31.536425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2c6ee7c-f4df-444e-ae8b-29669c0b5557]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.579095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d7becc2-2337-4c5d-bad7-60e89cdcd0e2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.633549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c2c59af-49af-45b1-b74f-764eae03f000]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 132665, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.673695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f203d07a-846b-4f14-b0fd-36669df1a894]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.693447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81ce7bda-3412-450c-9c5e-8f0ac0690158]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.719259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b47e7a23-162c-402c-ad6f-5dd2227314ae]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.760385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5400e307-1a74-420f-8686-4a7bb8ada8f3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.761169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:31.761613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:31.762062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:31.762343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:31.786115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:31.786809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef35a88e-3cf0-4358-8c7e-dd6032827ded]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.791424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cff19599-24a8-4435-95bc-21926900c199]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.792692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:31.829551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:31.831983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:31.832665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:31.833367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:31.833994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:31.900740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2e9d2a3-b7c0-4a87-845e-a870e8e7856a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.929323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[26533832-a01e-482f-9b96-e9e848a5f88b]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:31.929323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:31.929323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:31.929323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:31.929323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:31.929323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:31.929323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:31.929323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:31.929323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:31.929323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:31.929323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:31.929323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:31.929323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:31.931141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26533832-a01e-482f-9b96-e9e848a5f88b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:31.932695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:31.936212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:31.940224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d1cb071-00db-4265-bb56-ba18209798bb]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.941162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:31.941793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:31.942247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:31.942782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:31.944307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62c567b1-5511-4926-be2d-dfb5512f9bd0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.945598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[635361a7-929f-4982-8fd5-d2c46f25544b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.946938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a6aca01-b255-4ddf-a5fd-defe92f2809f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.990950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14e104c9-4db9-455a-be3e-ae338b0a8c16]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 272788, 'reachable_time': 31096, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 132802, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:31.996093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:31.998754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb50e493-a984-4bd1-b4a7-8724657c9474]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:32.000155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:29Z", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:32.007229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:32.025875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 5047d151-2cc9-4333-ae13-1bfa2875eddf. Wait time elapsed: 22.544 Jul 01 09:30:32.025875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 5047d151-2cc9-4333-ae13-1bfa2875eddf. Wait time elapsed: 22.545 Jul 01 09:30:32.025875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:32.044552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.072s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:32.045586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:32.047079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:32.047844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:32.048360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:32.054409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f98e7ee0-43d6-4833-a249-057fdf74e5fe]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.086632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[415f9671-a914-46c4-9391-cbfddd9652ea]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.103484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4456aca8-1d2b-4bf6-b381-de2c9e0a16cd]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.138678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15340d47-5876-4c3e-8895-b96ffb815250]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 132871, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.166967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59e9759f-f110-4408-9fdb-a8b52810a3d1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.193573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[140ef185-16ce-4e5e-b9c1-ee6da1dd97b8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.225529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96c7ab14-2bff-40eb-a06c-6075e7514f13]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.250708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2a11196-3c27-4a8c-9838-4e979a00a89e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.251799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:32.252236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:32.254814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:32.255279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:32.284509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:32.285083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd42d970-66ba-486b-9b78-1ccfce76f252]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.288041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05a8f3e8-5b22-418c-b72d-505c536cd8ef]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.289228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:32.334554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:32.336731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:32.337281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:32.337832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:32.338379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:32.381326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17ce858b-43ca-438b-a793-cd935593c223]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.447524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[eea408dd-e8dd-46dc-9bbe-39cf5ee8cf37]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:32.447524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:32.447524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:32.447524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:32.447524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:32.447524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:32.447524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:32.447524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:32.447524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:32.447524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:32.447524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:32.447524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:32.447524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:32.451205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eea408dd-e8dd-46dc-9bbe-39cf5ee8cf37]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:32.453360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:32.458006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:32.464849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0709b5f-4f4f-47d0-aab4-690fedbef70f]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.465970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:32.468245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:32.468704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:32.469309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:32.470595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bbdfcf8-e64a-4d17-a6cd-a7eb0244b2f8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.472895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fc66569-2bc2-4b9e-b981-a6c0e7243e1a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.475942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27002f0c-6726-4c2f-96be-ffb25cad4b84]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.524015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0a7bdf4-9d6b-4287-8a47-44c7a597b9e4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 272840, 'reachable_time': 27364, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 133037, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.530692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:32.531320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eea7a3a4-a8c7-400e-8af1-8b11b55352d0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:32.533187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:29Z", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:32.538824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:32.565655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b7af566d-370a-4752-a709-3059693f78e4. Wait time elapsed: 15.228 Jul 01 09:30:32.565655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b7af566d-370a-4752-a709-3059693f78e4. Wait time elapsed: 15.229 Jul 01 09:30:32.565655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:33.122900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c3c94111-b212-436f-ba01-6e8b6fd116d7 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:33.124195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 28accc35-2363-42ab-a7b0-ac873be0fa71. Wait time elapsed: 0.001 Jul 01 09:30:33.124440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 28accc35-2363-42ab-a7b0-ac873be0fa71. Wait time elapsed: 0.001 Jul 01 09:30:33.124893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:33.649822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.105s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:33.651237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:33.652774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:33.653294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:33.653580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:33.656620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f47e0f30-a47f-408b-a20e-fa34eac637db]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:33.680153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0563a414-20d5-4630-9d96-9e8da915c964]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:33.706004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b1ff9e1-8752-417c-844f-aa3efc4a883f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:33.748981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07ceaa74-9bc8-48c1-af06-9af4728e8005]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 133544, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:33.771715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[caccff83-c1a0-4976-b97a-963762a11be0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:33.796114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbcaa970-7467-47b7-876f-1f1d0d43f121]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:33.832591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ce2c312-7f44-4ce9-b59e-4eeb3fdcdd1e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:33.869366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9dd7636a-ec2c-4e32-ba0f-953e229b7242]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:33.870034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:33.870417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:33.870792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:33.871175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:33.897433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:33.898076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac77b64d-5a7a-4b42-9273-373d562a3595]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:33.902044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce9c4702-eba3-4a05-a468-be9656ca9d72]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:33.903338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:33.940766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:33.941816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:33.942423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:33.942880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:33.943658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:34.006904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab54f330-0509-4eab-81b3-be690a7ab7df]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:34.039942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b453f142-3a42-4965-9b95-02c113000d7c]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:34.039942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:34.039942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:34.039942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:34.039942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:34.039942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:34.039942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:34.039942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:34.039942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:34.039942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:34.039942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:34.039942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:34.039942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:34.041211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b453f142-3a42-4965-9b95-02c113000d7c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:34.041832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:34.044105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:34.052289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36504c6b-6a40-4723-a696-95a1e5aaff1a]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:34.053019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:34.053508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:34.053862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:34.054325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:34.060061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bc1f580-ef54-4d9a-8e36-24832a37cead]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:34.065656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef1f40fc-efbd-4003-8713-ae1e44084dac]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:34.067133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7a86b9d-4fb7-49c5-bbc3-3c4307f3bacc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:34.111577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fec4f71-48b7-456b-95d6-a2f656e2199f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 273001, 'reachable_time': 40008, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 133716, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:34.121606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:34.122588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7ccb3d7-e398-4042-b2e4-e89fc46120d3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:34.125826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:32Z", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:34.131056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:34.158083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id d37615a9-34b6-4d4d-b579-97954c960dd9. Wait time elapsed: 21.978 Jul 01 09:30:34.158083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id d37615a9-34b6-4d4d-b579-97954c960dd9. Wait time elapsed: 21.978 Jul 01 09:30:34.158083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:34.207074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-013ff361-750c-489a-b85a-be3242f603e9 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['c61a3a7a-d661-40a7-b816-e2c2b4ed2cba'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:34.208650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c61a3a7a-d661-40a7-b816-e2c2b4ed2cba, action 3, priority 1, update_id cc28f264-8c47-4125-bd1d-fb0eda21fa1f. Wait time elapsed: 0.001 Jul 01 09:30:34.209159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c61a3a7a-d661-40a7-b816-e2c2b4ed2cba, action 3, priority 1, update_id cc28f264-8c47-4125-bd1d-fb0eda21fa1f. Wait time elapsed: 0.001 Jul 01 09:30:34.209678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:34.478003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.353s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:34.478550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:30:34.480586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0aaaa1d-a51a-4c49-9dc8-f93974dda95a]: (4, ['qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:34.481317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:34.481628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:34.481889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id 28accc35-2363-42ab-a7b0-ac873be0fa71. Time elapsed: 1.358 Jul 01 09:30:35.766050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.624s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:35.766753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:35.767561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:35.768346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:35.769035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:35.771781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a96b45a1-824d-4295-8732-32e281f3f804]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:35.802924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cf7f56a-2321-484f-b369-f56329e32d87]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:35.851516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c6c8cac-5ab5-4a4d-97fd-b080d2370ac6]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:35.929644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[feb01c28-2b83-4b93-b340-635207f7da62]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 134404, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:35.969400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03b46b8b-f79f-425b-8e91-da29ae14b1ef]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.011795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c027bba-6d04-4c08-9fe5-4cda376b7452]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.057003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da547ea1-d26e-488c-921e-e33b392052f2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.093096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9412f168-0b15-4424-9584-e0b7bd749ccc]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.094010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:36.094393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:36.094759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:36.095120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:36.119939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:36.120594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b345142-6a6b-417c-ab35-56b15bf205dc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.125474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84780a5d-2051-446c-b522-982ffbe69986]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.126670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:36.165979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:36.166789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:36.167298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:36.168439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:36.169226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:36.239499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5fd11e0-44a8-4537-be0a-b9166f52b477]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.280651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d28adf33-0ac5-4f67-b669-c876405139f8]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:36.280651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:36.280651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:36.280651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:36.280651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:36.280651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:36.280651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:36.280651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:36.280651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:36.280651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:36.280651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:36.280651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:36.280651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:36.282514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d28adf33-0ac5-4f67-b669-c876405139f8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:36.283686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:36.287256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:36.291340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea3abc48-84c6-4d64-8480-a88ad24caffd]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.297263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:36.297953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:36.298406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:36.299072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:36.300978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfdf4465-8a82-4b24-af65-194f0837ee70]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.305844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53a0c2dd-2ecc-4c9f-94c3-2cf0aed41b66]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.308124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b6f406d-3fcb-4b72-9cc9-54650f88873e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.337191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.127s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:36.338050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router c61a3a7a-d661-40a7-b816-e2c2b4ed2cba was not found. Performing router cleanup Jul 01 09:30:36.339814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92d15d17-f16f-409c-a70b-fa800916a109]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.340647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:36.341224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:36.341766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for c61a3a7a-d661-40a7-b816-e2c2b4ed2cba, update_id cc28f264-8c47-4125-bd1d-fb0eda21fa1f. Time elapsed: 2.133 Jul 01 09:30:36.345499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0bf75cb-7fcf-48d2-b3eb-546cb86824cf]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 273219, 'reachable_time': 32364, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 134593, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.352864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:36.353372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32ccbbb0-40c7-4e2f-b559-34c0c557f1ea]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:36.354868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:32Z", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:36.358667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3 Jul 01 09:30:36.378634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b7af566d-370a-4752-a709-3059693f78e4. Wait time elapsed: 19.041 Jul 01 09:30:36.378634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b7af566d-370a-4752-a709-3059693f78e4. Wait time elapsed: 19.042 Jul 01 09:30:36.378634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:37.440869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.419s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:37.441421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:37.442637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:37.443264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:37.443697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:37.444907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ad01ef8-ea5c-4fc5-97d7-3e68bace1c64]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.524357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7284544-e4e8-4d93-9f78-dd14a19414b2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.583236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43bbf509-f4b7-421a-b8e4-2f377adc6355]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.661330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eeb4d06f-77bf-404f-8478-1b9d068a74b2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 135100, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.695285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5df7b341-3e42-4554-893a-a3ea7661f88d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.740219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.381s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:37.740680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:37.742915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:37.745832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:37.746350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:37.747388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[028a72c4-173c-49f1-9bd3-1ce9e6572a1e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.748587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53bcea5e-71e9-4312-8b88-b180fd86de77]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.777894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e497648-56f8-4896-8b5e-3aba4e78aa8a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.788817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e97f016-3988-4324-8b70-6e59997b48da]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.811581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0b83004-b7ec-418d-9c0d-62cbe6229d44]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.851021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c89e1bc1-41dd-4849-bd1e-5d429f52d408]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.851892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:37.852302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:37.852822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:37.853361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:37.881693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4ebfe25-c504-4bd9-8a65-3f8db988613f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 135190, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.883956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:37.884486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[369f2d10-2392-480a-b00a-bf52892fc2e5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.890647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[091dc6e8-4f34-4bf1-b0ac-d63a36004d44]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.892294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:37.927789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:37.928832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:37.930041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:37.930806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:37.931607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:37.984042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76db5c73-7bbc-4823-a81a-179c936e6d3e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:37.998682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b0feb02-8c9a-4040-8e4a-67d31d353049]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.030726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f18d8e08-d1c5-423f-9345-377363712d6c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.051862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[47d755d7-f951-4e9b-9520-d149d22fc684]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:38.051862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:38.051862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:38.051862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:38.051862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:38.051862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:38.051862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:38.051862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:38.051862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:38.051862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:38.051862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:38.051862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:38.051862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:38.055661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47d755d7-f951-4e9b-9520-d149d22fc684]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:38.056766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:38.059768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:38.066367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[edf022a8-f237-4d89-98b6-ee87847117a3]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.067389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:38.067890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:38.068560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:38.069065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:38.070602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4a0c1a9-471b-46af-bb93-52094ca398e8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.071914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44e89f76-f266-4ae8-b835-2c32c271bbd7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.074514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[146a5bae-b74a-47fe-a027-d3ff669f9a5c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.074998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95410f2d-24f5-4bf6-8e9e-60f9d22dfc77]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.102215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe13b8dd-097a-41c1-977e-7c4df8fdf2ab]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 273392, 'reachable_time': 19532, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 135297, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.115268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:38.115727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57d2e22a-5dde-42d2-ad95-728edea2f58b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:38.116923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:33Z", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:38.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:38.142648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 7b94701f-1773-4678-a963-a5ac13db9ced. Wait time elapsed: 18.117 Jul 01 09:30:38.142648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 7b94701f-1773-4678-a963-a5ac13db9ced. Wait time elapsed: 18.118 Jul 01 09:30:38.142648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:38.152112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79861bdf-242d-4f93-964d-d1772a5e4ea6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.152623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:38.152897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:38.153162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:38.153442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:38.197668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:38.198214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10d24c72-0a15-47b7-ace6-0bda0ff272f4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.204371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfc1bbe5-bfe6-4219-85c5-7cba5a32ad6b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.207014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:38.245066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:38.246044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:38.248159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:38.250038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:38.251096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:38.296008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f883af1-d92f-41df-9dda-c6a4d12d41dc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.347405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[95d64341-2664-41a3-8736-347c250986ac]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:38.347405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:38.347405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:38.347405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:38.347405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:38.347405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:38.347405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:38.347405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:38.347405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:38.347405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:38.347405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:38.347405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:38.347405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:38.350760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95d64341-2664-41a3-8736-347c250986ac]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:38.351857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:38.355440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:38.359568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11f08b5c-c03d-4f62-af95-aefc90c04bd2]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.361812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:38.362494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:38.362943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:38.363774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:38.365080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8366ea3d-5d48-42e4-88d6-968ec68ceda7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.366280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[080cef1e-6232-47ab-a57a-9f99f8c496a7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.369621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cebf2c37-2e01-4eea-b499-fdca9c8df4a1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.460811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6bda86f3-78dd-45ea-b72e-2b6ea5fffa1c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 273413, 'reachable_time': 29240, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 135424, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.470410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:38.470984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e759a685-0bad-4311-9b49-639976136410]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:38.472341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:36Z", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:38.478421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:38.501944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b7af566d-370a-4752-a709-3059693f78e4. Wait time elapsed: 21.178 Jul 01 09:30:38.501944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b7af566d-370a-4752-a709-3059693f78e4. Wait time elapsed: 21.178 Jul 01 09:30:38.503309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:39.603795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-2dbaf943-febc-492c-9328-f41d14b19d3b tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['c61a3a7a-d661-40a7-b816-e2c2b4ed2cba'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:39.605593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c61a3a7a-d661-40a7-b816-e2c2b4ed2cba, action 3, priority 1, update_id 72437e0b-3257-45fe-967c-e3d2c854adc3. Wait time elapsed: 0.001 Jul 01 09:30:39.606218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c61a3a7a-d661-40a7-b816-e2c2b4ed2cba, action 3, priority 1, update_id 72437e0b-3257-45fe-967c-e3d2c854adc3. Wait time elapsed: 0.001 Jul 01 09:30:39.606751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:40.295167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.793s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:40.296317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:40.298028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:40.298524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:40.299128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:40.303291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4615fd96-740f-472c-bc3d-50e1d4beb561]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.331877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5042985-8f4d-4af8-a892-e65084fab65a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.368174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55e7f822-4075-455d-99ec-7db383b9ff56]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.439026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05bf6d08-5782-40dc-9fcf-e76f8529d7c6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 136148, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.458496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74f30b3d-0e45-4566-bc74-64c327c78bf9]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.504421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-02fff9e8-6309-4284-b8b8-8cbdfc30a4d4 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:40.505596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22bd5dcc-3ff1-4fa1-b1ef-49ff1d2de5dc]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.507515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 8529a975-c739-4fe1-8c72-02e88a613e4d. Wait time elapsed: 0.002 Jul 01 09:30:40.507893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 8529a975-c739-4fe1-8c72-02e88a613e4d. Wait time elapsed: 0.003 Jul 01 09:30:40.508469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:40.538071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4538debb-3e80-4602-a21d-13f6a79551d1]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.597712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbad662d-197f-42a1-bf4f-afe811cd0cfb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.598676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:40.599102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:40.599656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:40.600029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:40.636197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:40.637244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f558c588-c647-4d95-a275-c471a2ef32c9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.640872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2de87061-0b6c-4348-824d-56e546c874b0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.642943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:40.669270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:40.670883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:40.671582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:40.672342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:40.672955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:40.734066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b864420e-b723-41b1-886f-2db5b6106a53]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.802017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.293s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:40.802836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:30:40.805070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7d559be2-cac1-4f5b-a02f-5869d8183d00]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:40.805070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:40.805070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:40.805070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:40.805070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:40.805070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:40.805070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:40.805070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:40.805070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:40.805070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:40.805070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:40.805070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:40.805070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:40.807561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d559be2-cac1-4f5b-a02f-5869d8183d00]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.808240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfc61078-c952-4c58-85e3-1e1f07f369b5]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:40.809920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:40.814100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:40.818915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:40.819197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:40.819509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id 8529a975-c739-4fe1-8c72-02e88a613e4d. Time elapsed: 0.312 Jul 01 09:30:40.819953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12d5b736-22b8-4aa4-ae4b-284681685167]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.821173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:40.821500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:40.821756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:40.822099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:40.826487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9fd296a9-991c-47b0-8575-fa14143f5dcb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.829388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d32463fd-59b4-4076-8fdc-84342d9de0e1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.830509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e49bbd13-c3dc-4f30-bf73-a1448c7f8d38]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.863416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40ea3355-42a1-4a11-b36b-98bbd22961f7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 273670, 'reachable_time': 34664, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 136346, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.878560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:40.878850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ad9e60f-a63c-4d7a-9b8e-43b956ed5f4f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:30:40.880102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:36Z", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:40.884148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:40.906666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3 Jul 01 09:30:41.024556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.417s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:41.026993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router c61a3a7a-d661-40a7-b816-e2c2b4ed2cba was not found. Performing router cleanup Jul 01 09:30:41.031021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[776c1142-783e-410b-83c6-ed405e23f850]: (4, ['qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:41.034743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:41.035305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:41.035923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for c61a3a7a-d661-40a7-b816-e2c2b4ed2cba, update_id 72437e0b-3257-45fe-967c-e3d2c854adc3. Time elapsed: 1.430 Jul 01 09:30:41.103682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-f7fcd713-20bd-4f2c-ad3d-58de28814ade tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got router deleted notification for c61a3a7a-d661-40a7-b816-e2c2b4ed2cba {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:30:41.104583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c61a3a7a-d661-40a7-b816-e2c2b4ed2cba, action 1, priority 1, update_id 38f8cf8c-441f-456e-9d1f-b70800932cbe. Wait time elapsed: 0.001 Jul 01 09:30:41.104830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c61a3a7a-d661-40a7-b816-e2c2b4ed2cba, action 1, priority 1, update_id 38f8cf8c-441f-456e-9d1f-b70800932cbe. Wait time elapsed: 0.001 Jul 01 09:30:41.105077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router c61a3a7a-d661-40a7-b816-e2c2b4ed2cba was not found. Performing router cleanup Jul 01 09:30:41.106122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76e6d1f3-a91c-48e3-91d7-9a9c9a10315e]: (4, ['qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:41.106620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:41.108068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:41.108302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for c61a3a7a-d661-40a7-b816-e2c2b4ed2cba, update_id 38f8cf8c-441f-456e-9d1f-b70800932cbe. Time elapsed: 0.004 Jul 01 09:30:41.264931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-944c417e-514f-4ffd-a767-0b11e3a60049 tempest-ServersTestJSON-1669246664 tempest-ServersTestJSON-1669246664-project] Got routers updated notification :['4da8db80-0643-4392-ad3e-03f5c3888347'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:41.270830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 1, update_id a9818be2-aba5-4cfc-8e5c-79970a660672. Wait time elapsed: 0.005 Jul 01 09:30:41.277383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 1, update_id a9818be2-aba5-4cfc-8e5c-79970a660672. Wait time elapsed: 0.006 Jul 01 09:30:41.278404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:44.833838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 6.692s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:44.834908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:44.836411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:44.836969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:44.837497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:44.838986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c32c13a2-9d7a-44f1-96cd-25e0d8d937cd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:44.867289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[418374b9-dc02-4fb3-ba3a-b8755285168e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:44.891166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c7864bd-db2a-44c2-a273-9a4efe8b4b99]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:44.932022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b899edd-e751-478b-85a2-1bedbc855a0f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 136736, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:44.953938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63148dc1-e43f-4288-b9d4-a43dcf70d1fa]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:44.980773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6fa5151c-8d6e-429d-b03c-15123bf760b5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.016243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e4fb1f8-c897-49d7-8966-54efd31f0c9b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.023294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-0c73023a-6823-44dd-bec7-ee33e253b754 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['9b96a59e-ed9b-4059-a7ba-61d79a76a3ac'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:45.024614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, action 3, priority 1, update_id 7c89157d-9b00-43f6-9c30-c08cda6a9270. Wait time elapsed: 0.001 Jul 01 09:30:45.025075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, action 3, priority 1, update_id 7c89157d-9b00-43f6-9c30-c08cda6a9270. Wait time elapsed: 0.001 Jul 01 09:30:45.025590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:45.073929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e87902e1-c979-4bb2-b364-7ecb7d8cf7cf]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.074832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:45.075320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:45.076452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:45.077251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:45.111201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:45.111729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b670dfbe-dd9f-40ce-9ae5-c0301b9e3d10]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.114410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01f0f0ab-0b86-4fcb-aabc-ff0d475dd915]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.115469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:45.162616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:45.164068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:45.164460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:45.165030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:45.165860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:45.221208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.195s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:45.221799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac was not found. Performing router cleanup Jul 01 09:30:45.223761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[167a7f1f-c9bb-410d-9116-f1692d5cb335]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.225801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:45.226593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:45.227062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, update_id 7c89157d-9b00-43f6-9c30-c08cda6a9270. Time elapsed: 0.202 Jul 01 09:30:45.232806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d51bc8e2-679e-44db-9fa9-37677b1e58c5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.284026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[56d5ce8c-d617-4caa-9ff0-8384b5a02d31]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:45.284026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:45.284026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:45.284026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:45.284026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:45.284026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:45.284026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:45.284026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:45.284026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:45.284026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:45.284026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:45.284026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:45.284026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:45.285140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56d5ce8c-d617-4caa-9ff0-8384b5a02d31]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:45.285240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:45.287062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:45.289472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2d9ce48-1ac6-46b8-8716-1e58f33fdee6]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.289472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:45.289472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:45.289472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:45.289472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:45.291914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aec592b6-623c-4a63-b194-a47e27cded37]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.297679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1ca124d-9338-41e4-9614-45011bba84ea]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.298814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3aff8db6-24a5-470d-8f40-f3b0c7e24fbb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.325341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad0719f4-f4ef-4f5d-88b9-cf8f1f252f33]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 274119, 'reachable_time': 35512, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 136757, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.337761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:45.338163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03b10426-f436-40cd-b33d-299f19b5cb76]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:45.339626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:37Z", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:45.343779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 5047d151-2cc9-4333-ae13-1bfa2875eddf. Wait time elapsed: 35.864 Jul 01 09:30:45.369647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 5047d151-2cc9-4333-ae13-1bfa2875eddf. Wait time elapsed: 35.864 Jul 01 09:30:45.369647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:46.198006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.919s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:46.198622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:46.200392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:46.201268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:46.201738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:46.203025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64a4744d-31d8-43b6-afa5-f7796b290715]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.219808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24a4c7ed-933b-4665-afbf-e023d792367d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.253499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d91d5de2-eb95-4d5d-ac99-47652f8c3daf]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.299479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3285b8dc-6100-4564-a71a-6d2eed04c2f2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 136760, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.324637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8616a400-9171-4308-99b3-a64d675ea0ca]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.351661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b35a29f-5e59-4286-a020-70367919e777]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.377768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a47555a-bc90-4b69-aa49-37d827cf5abe]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.445402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87c2a02f-2125-4fc7-9fb1-fe1941aabf45]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.446297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:46.446800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:46.447276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:46.447804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:46.483385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:46.484715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37421998-2be1-4d77-860e-01492d36e204]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.488550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b0e2e7c-5de8-4efe-b702-83b225258902]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.491346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:46.524393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:46.525328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:46.525939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:46.526576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:46.527147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:46.584525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8237e090-3743-4602-aa18-f2cb8fba58b9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.618509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bbdb75b0-bb6f-48c6-a197-fa6b06a875d8]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:46.618509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:46.618509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:46.618509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:46.618509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:46.618509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:46.618509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:46.618509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:46.618509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:46.618509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:46.618509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:46.618509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:46.618509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:46.620083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbdb75b0-bb6f-48c6-a197-fa6b06a875d8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:46.621895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:46.624156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:46.630972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[172d3822-f049-4388-adec-bac069d8218a]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.632267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:46.632649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:46.632911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:46.633307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:46.635762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d066a6fa-84a8-4961-97fd-b87c588c488b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.637518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bed35a98-6cf5-4c69-abaa-86c4c72aa371]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.639626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df35f849-3a73-4053-a71a-431bb374f9fe]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.671659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad71fe1b-ad4b-4c77-9b20-fdad15adca92]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 274256, 'reachable_time': 36600, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 136779, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04d87282-c23e-4b64-adb2-197d418260c9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:30:46.720079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:36Z", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id a9818be2-aba5-4cfc-8e5c-79970a660672. Wait time elapsed: 5.413 Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id a9818be2-aba5-4cfc-8e5c-79970a660672. Wait time elapsed: 5.413 Jul 01 09:30:46.728582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:47.691856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-e2bfb70d-0aa6-44ff-943c-5a8567c08460 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['fad334a2-30d2-4eb6-b2ec-5d949314fe3c'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:47.693803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fad334a2-30d2-4eb6-b2ec-5d949314fe3c, action 3, priority 1, update_id 3e924585-902e-4b8a-9090-ad2aa94bbfe0. Wait time elapsed: 0.001 Jul 01 09:30:47.694174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fad334a2-30d2-4eb6-b2ec-5d949314fe3c, action 3, priority 1, update_id 3e924585-902e-4b8a-9090-ad2aa94bbfe0. Wait time elapsed: 0.001 Jul 01 09:30:47.694597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:47.808710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.130s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:47.808984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:47.814056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:47.814839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:47.820496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.006s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:47.827751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b7a8824-4e81-4290-94da-c4345e50fe15]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:47.852612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0125b2ed-50a9-4c02-aea3-0a8c0356c38c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:47.896853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fd1ec8e-6dbc-4396-b917-585e9c5bb33e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:47.938402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61560a93-6fe1-4233-ac93-4cf1b273d619]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 136799, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:47.963675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[235382f5-dc9e-4b27-bb58-19f7bce2ed64]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.000400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f8f6c5f-aa45-46ee-9c9b-b8c2843331ef]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.038512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0daf6d8-31fa-429d-bada-eb1bbc15cb83]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.083153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51a16711-7e09-4f28-a5e1-7f942505b007]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.084875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:48.085405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:48.085864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:48.086820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(acb72bdc-7be7-4e50-a658-28f451b84138) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:48.115743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:48.116933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6b7b5f3-a65a-478c-bddc-e81c5280e3dc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.123518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abda83ef-fb83-4f34-92d6-d94d1eebf7d8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.125669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-acb72bdc-7b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:48.152069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-acb72bdc-7b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:48.153370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:48.154497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:48.154947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-acb72bdc-7b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:48.155433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-acb72bdc-7b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'acb72bdc-7be7-4e50-a658-28f451b84138', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b7:82:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:48.192405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fe6d02f-e298-4132-8ad7-de9c34161924]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.244265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6f2a1284-be5a-47f3-9b1a-64506cb8f122]: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:48.244265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:48.244265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:48.244265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:48.244265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:48.244265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:48.244265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:48.244265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:48.244265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:48.244265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:48.244265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:48.244265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:48.244265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:48.245289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f2a1284-be5a-47f3-9b1a-64506cb8f122]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:48.245551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:48.247126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:48.249747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40842677-2071-49dc-a07f-39236451e641]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.249747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:48.249747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:48.249747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:48.250044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:48.251132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ccf0f522-815b-4bfe-ba58-c8624a1f3c0b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.252759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c47b74f5-9ceb-4730-a46c-17a1ae45e357]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.253778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fed0fb82-ab4d-40fe-9679-9eb08a6432f5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.277694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2eafbfba-3f1a-4852-9f86-f40062a12cfa]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 274420, 'reachable_time': 40772, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 136817, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.279847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:48.280285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43134e94-4997-46f7-9d2e-eae5d9bbfc89]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-acb72bdc-7b not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:48.282493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:03Z", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "acb72bdc-7be7-4e50-a658-28f451b84138", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c21f3643-3e41-4db0-97a0-4a6ecc213ff8", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b7:82:f6", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 464, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:36Z", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a559b19e-c29e-4b4d-a21d-9d0e80f88861", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:48.286503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:48.297236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id a9818be2-aba5-4cfc-8e5c-79970a660672. Wait time elapsed: 7.027 Jul 01 09:30:48.297236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 2, update_id a9818be2-aba5-4cfc-8e5c-79970a660672. Wait time elapsed: 7.028 Jul 01 09:30:48.297236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:48.837895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.143s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:48.838809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router fad334a2-30d2-4eb6-b2ec-5d949314fe3c was not found. Performing router cleanup Jul 01 09:30:48.840022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b4a1611-f5a8-48af-a4a7-4d8c14523311]: (4, ['qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:48.840699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:48.841164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:48.841692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for fad334a2-30d2-4eb6-b2ec-5d949314fe3c, update_id 3e924585-902e-4b8a-9090-ad2aa94bbfe0. Time elapsed: 1.148 Jul 01 09:30:49.379223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.086s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:49.379725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:49.380839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:49.381302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:49.381721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:49.382955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbad53a7-3237-4710-8517-06469b1ba466]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.405615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35f3482d-5ca9-4d51-8324-4e9200ac1508]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.468012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5c9096c-9b8e-4d4a-a060-9bca71fa3aad]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.519412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dca2d7df-0065-4359-8602-315b402cdd2c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 136835, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.543382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.202s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:49.544126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:49.545746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:49.546275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:49.546732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:49.549482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ca39595-69ae-4730-9056-e3fe78d20682]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.566611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17243b64-ed17-4572-93c7-82be66431a68]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.570781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76e73817-3101-4e65-8d1a-a0e4b93ae7c0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.610758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46dcb128-9252-4515-883e-49f3076ba2f4]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.611758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0acfe82-86e3-4d50-98ce-d2c84f407ac0]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.648818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6db2e40-a8f8-4dfb-9f98-4004d62acaf8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.654990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c2e5c52-a774-4cc2-bb12-1de1db06309c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 136842, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.674708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9aee70c4-d87e-4d5e-84af-9a8774ee04c6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.675966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:49.676402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:49.676801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:49.684968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e8b38ba-1b0c-46c3-9909-b9519aa20a2e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.713582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27c05e26-b8c3-4e5d-8820-b1451a4fd208]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 274578, 'reachable_time': 19908, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 136847, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.715037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:49.715808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:49.716942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:30:49.717661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:30:49.718133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:49.718595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:49.719071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '37d4d8f8-7374-4b3b-b030-66d7cccb36f0', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:c8:0e:04', 'admin_state_up': True, 'status': 'ACTIVE', 'device_id': '4da8db80-0643-4392-ad3e-03f5c3888347', 'device_owner': 'network:router_gateway', 'standard_attr_id': 407, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.113', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:25:54Z', 'updated_at': '2022-07-01T09:26:09Z', 'revision_number': 6, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-37d4d8f8-73), ns(qrouter-4da8db80-0643-4392-ad3e-03f5c3888347) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:30:49.720440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7173a97d-8fe6-4d46-a517-3ee01224e734]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.745199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:49.745775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45f3f58b-61e4-42c8-8c89-7b90d3cd8a91]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.750652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4483d020-085f-4c5c-b7d6-7fa9da70659a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.752040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[972250a6-84c4-44e8-81a2-612a17c93991]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.752839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-37d4d8f8-73, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:49.780646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-37d4d8f8-73, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:49.782734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-37d4d8f8-73, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:49.783200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-37d4d8f8-73, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:49.783747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-37d4d8f8-73, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:49.784377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-37d4d8f8-73, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37d4d8f8-7374-4b3b-b030-66d7cccb36f0', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:0e:04'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:49.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fcca19c-319f-4325-9ef9-5faf6fdaf300]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.809704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:49.810703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:49.810703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:49.811303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:49.822188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40c730bc-f22d-4dc8-b65b-99b8c32a8bc9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.864047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:49.864669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a39ad2a4-066a-4e97-ad81-e2736603fdc3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.866989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5b905b6-affa-4b03-b603-5f8739dda0a5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.868261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:49.890377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b64d36c2-203b-4cb3-a505-8e99be9e02c1]: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:49.890377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:49.890377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:49.890377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:49.890377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:49.890377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:49.890377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:49.890377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:49.890377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:49.890377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:49.890377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:49.890377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:49.890377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:49.891132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b64d36c2-203b-4cb3-a505-8e99be9e02c1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.891198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:30:49.891838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:30:49.900652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:49.902138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:49.902710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:49.903756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:49.904891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:49.941018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1702cef-0194-4555-89d7-0cb9bcbb35c8]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:49.953955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78a4df4a-5670-486b-aa7d-fca3ea77bbc9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.007649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9e693a8e-6266-4be4-8b06-220c33562a43]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:50.007649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:50.007649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:50.007649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:50.007649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:50.007649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:50.007649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:50.007649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:50.007649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:50.007649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:50.007649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:50.007649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:50.007649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:50.009389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e693a8e-6266-4be4-8b06-220c33562a43]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:50.010483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:50.012753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:50.016737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e79f212-1892-49e0-8a13-4660a32f179a]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.018329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:50.018837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:50.019244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:50.019900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:50.021430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[770b5b91-3b33-427c-9946-fa31a31c770c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.022478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3401711b-b5ff-4a5d-957f-fe7c822d6536]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.023629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8841229e-3a3b-4a2d-8660-82db9d883b5c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.048303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f95f6d5-373f-4a0d-92be-bc6743c4490c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 274591, 'reachable_time': 28828, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 136877, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.057776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:50.058479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06fe1c00-940a-47b4-9a54-1793fea6b296]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:50.060518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:47Z", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:50.065003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:50.080103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 7b94701f-1773-4678-a963-a5ac13db9ced. Wait time elapsed: 30.054 Jul 01 09:30:50.080103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 7b94701f-1773-4678-a963-a5ac13db9ced. Wait time elapsed: 30.054 Jul 01 09:30:50.084241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:50.194847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81f6da44-43a7-43ce-b6ae-a4c8cf285431]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.217355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc109974-a592-4570-8076-c03d08ba4fde]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.270181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-5996ff35-d1c8-47ae-b00a-881b8d105cab tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['fad334a2-30d2-4eb6-b2ec-5d949314fe3c'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:50.270181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fad334a2-30d2-4eb6-b2ec-5d949314fe3c, action 3, priority 1, update_id 252039da-13c2-4a68-9e73-38fc207fec37. Wait time elapsed: 0.001 Jul 01 09:30:50.277929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fad334a2-30d2-4eb6-b2ec-5d949314fe3c, action 3, priority 1, update_id 252039da-13c2-4a68-9e73-38fc207fec37. Wait time elapsed: 0.001 Jul 01 09:30:50.278351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:50.351998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8f7498d1-41e0-4f46-ba21-fcf9590edafe admin admin] Got routers updated notification :['4da8db80-0643-4392-ad3e-03f5c3888347'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:50.426858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.148s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:50.427418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router fad334a2-30d2-4eb6-b2ec-5d949314fe3c was not found. Performing router cleanup Jul 01 09:30:50.429941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c29089b-9c5f-4a1f-9cd9-e1a5336d2d8d]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.430670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:50.431115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:50.432136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for fad334a2-30d2-4eb6-b2ec-5d949314fe3c, update_id 252039da-13c2-4a68-9e73-38fc207fec37. Time elapsed: 0.166 Jul 01 09:30:50.448273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ef2d85a-977d-48b9-b234-2bf279efccd5]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.448954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:30:50.449429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:30:50.449856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:30:50.450476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" released by "process_external" :: held 0.733s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:50.451920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:50.454738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:50.458509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abefc0fc-9937-4c9b-b03d-65fe3318e826]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.460011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:50.460475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:50.460869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:50.461399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:50.462719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0eedd56d-066c-4c52-a30c-aa3ddb136225]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.463904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73edf4b3-dea8-42fe-96e5-6759adb24ad8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.465471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fefc400-b04c-44ab-b516-3249c422ab59]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.494218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9af176b5-22be-4fa5-a9ea-867ba42a05df]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 274578, 'reachable_time': 19908, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 136880, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.505389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:50.505897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b041d65c-2630-4129-97fe-e969d374dd5e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4da8db80-0643-4392-ad3e-03f5c3888347: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-37d4d8f8-73 not found in namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347. Jul 01 09:30:50.508356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestJSON-449308510-router", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113" Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:48Z", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "f55c06bf5a7d44aeb5879b9c646b158e", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37d4d8f8-7374-4b3b-b030-66d7cccb36f0", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:0e:04", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4da8db80-0643-4392-ad3e-03f5c3888347", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 407, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.113", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:09Z", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:30:50.511946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:50.518954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 1, update_id ab0dd2f7-7262-4b4d-b5f4-cff3fb7e004d. Wait time elapsed: 0.165 Jul 01 09:30:50.519505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 3, priority 1, update_id ab0dd2f7-7262-4b4d-b5f4-cff3fb7e004d. Wait time elapsed: 0.165 Jul 01 09:30:50.519980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:50.795911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-86a5d134-baf8-42e1-b60d-1dd5888d0378 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got router deleted notification for fad334a2-30d2-4eb6-b2ec-5d949314fe3c {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:30:50.797627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fad334a2-30d2-4eb6-b2ec-5d949314fe3c, action 1, priority 1, update_id 64a93ab2-b5fe-42c3-865c-03fb0da4973c. Wait time elapsed: 0.001 Jul 01 09:30:50.798163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fad334a2-30d2-4eb6-b2ec-5d949314fe3c, action 1, priority 1, update_id 64a93ab2-b5fe-42c3-865c-03fb0da4973c. Wait time elapsed: 0.002 Jul 01 09:30:50.798551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router fad334a2-30d2-4eb6-b2ec-5d949314fe3c was not found. Performing router cleanup Jul 01 09:30:50.799870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38de4426-4644-4522-974f-3286be72618f]: (4, ['qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:50.800786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:50.801221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:50.801603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for fad334a2-30d2-4eb6-b2ec-5d949314fe3c, update_id 64a93ab2-b5fe-42c3-865c-03fb0da4973c. Time elapsed: 0.004 Jul 01 09:30:50.963199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-d82b26f2-d6e8-4028-8ebd-cb7c4162616c tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['9b96a59e-ed9b-4059-a7ba-61d79a76a3ac'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:50.969904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, action 3, priority 1, update_id f3a2d38d-05ad-4723-b157-2fb7ffdf5554. Wait time elapsed: 0.006 Jul 01 09:30:50.970459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, action 3, priority 1, update_id f3a2d38d-05ad-4723-b157-2fb7ffdf5554. Wait time elapsed: 0.006 Jul 01 09:30:50.971151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:51.096160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.124s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:51.096425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac was not found. Performing router cleanup Jul 01 09:30:51.097887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a95ee0b-f206-4998-8642-1cfd7c884634]: (4, ['qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:51.098663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:51.098934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:51.099183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, update_id f3a2d38d-05ad-4723-b157-2fb7ffdf5554. Time elapsed: 0.129 Jul 01 09:30:52.788268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.268s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:52.788862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4da8db80-0643-4392-ad3e-03f5c3888347 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:52.789566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:52.789673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:52.789911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:52.791880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[633b9c9d-2438-4e65-932d-3877730c75d7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:52.843261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed77ddbb-698b-43aa-b11c-3926dd20b47b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:52.889655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e4f0621-998c-41e9-a052-604e7e1c1a22]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:52.923694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef5ed96e-26f8-449f-be17-daed273f9f0f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 136883, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:52.945495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3878f29d-be5e-4f39-ac90-fd605b9bb9f5]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.009669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-1083d296-4a74-4e48-ad2f-e50be0f6d755 admin admin] Got router deleted notification for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:30:53.011045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[329ba740-5c3f-4bf1-9b57-3563bf10da35]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.053810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44e9e19b-7c09-4758-adcf-2ecaa1b67368]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.093000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74ad4f07-f547-4db9-976f-c76f12ba5c8f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.096037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:53.097050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:53.097684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:53.143505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f439eab-8bb8-43e1-8fb4-ec0958d87634]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 274918, 'reachable_time': 18968, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 136889, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.147300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:53.147944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:53.149476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:30:53.149969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:30:53.187512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d743012-a432-43b1-b005-9610ec0d2566]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 274918, 'reachable_time': 18968, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 136890, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.189731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:30:53.190419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:30:53.219945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b815823-3c25-4994-a87e-fddc1dfafb4c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.400566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58d29e57-13b3-4cf5-af5d-e8683a05a3f4]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.455163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ff6d5a5-84b5-4663-902c-6f23827e7ee3]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.473087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-513c5d96-ae01-4176-a26e-9e73d5a12690 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:53.474340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id f3ea2fc4-aca5-4f97-8d17-e60bb717a165. Wait time elapsed: 0.001 Jul 01 09:30:53.474857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id f3ea2fc4-aca5-4f97-8d17-e60bb717a165. Wait time elapsed: 0.001 Jul 01 09:30:53.475285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:53.597833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.514s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:53.598309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:53.599470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:53.600007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:53.600455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:53.602170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f915dbd-fbf2-4281-a7b0-2caf3b2405e8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.620599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1bc76fd3-0efc-4b08-ae2c-7f801a1bf4ed]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.654396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa762850-d670-4993-ab13-9ba78d494211]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.713242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dcc3fd64-bb15-456f-8bc7-795843c761d7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 136899, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.725431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7695b5a-2d9c-4864-b2d1-6dfe45d94279]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.726661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:30:53.728243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:30:53.728722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:30:53.729241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" released by "process_external" :: held 0.579s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:30:53.729868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:30:53.730824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" acquired by "process_address_scope" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:30:53.733116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:30:53.734255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:30:53.761917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d030228f-a285-46bd-acd9-1f602e34b33a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.778560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e901395b-2f19-48e6-a73a-544868f7789c]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:30:53 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:30:53 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:30:53 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:30:53 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:30:53 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:30:53 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:30:53 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:30:53 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.813522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03677e68-2ce3-4725-83a3-3092e8621f6e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.827562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47aecc88-8e20-42ee-9461-837bcc4f955d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.849461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85152238-babc-43ca-a9a4-3475c841dd27]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.860491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bb00765-928e-489a-8382-4ff1474654be]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:30:53 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:30:53 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:30:53 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:30:53 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:30:53 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:30:53 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:30:53 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:30:53 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.895604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e44d6df3-6ebf-4b01-ac87-82056dac46b5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.898959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:53.900486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:53.900910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:53.901325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:53.928890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1002f988-6662-407e-8115-c0b91dac9977]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.930287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:30:53.930801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:30:53.931230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:30:53.931797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" released by "process_address_scope" :: held 0.201s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:30:53.932409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:53.933052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:30:53.933485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:30:53.934183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:30:53.936648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:30:53.953452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-513c5d96-ae01-4176-a26e-9e73d5a12690 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['9b96a59e-ed9b-4059-a7ba-61d79a76a3ac'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:53.954845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:53.955291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0aaaa612-e536-4b4e-a6b5-e40094528cb5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.959511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, action 3, priority 1, update_id b7b427a9-1603-406b-a30c-bc4835cf0115. Wait time elapsed: 0.005 Jul 01 09:30:53.960005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, action 3, priority 1, update_id b7b427a9-1603-406b-a30c-bc4835cf0115. Wait time elapsed: 0.006 Jul 01 09:30:53.960508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:53.976280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8eb8c770-dd49-4f25-805a-ba296621a36e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:53.977675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:54.000344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1351f113-8899-4248-be51-be067eaba370]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:30:53 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:30:53 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:30:53 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:30:53 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:30:53 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:30:53 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:30:53 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:30:53 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.015700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:54.016551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:54.017189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:54.017609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:54.017998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:54.052657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7abd88b1-a3f6-4b7c-a49f-8566d5e68f71]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.105681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ce8f16e3-ef2d-46ef-9259-8b27100c0984]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:54.105681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:54.105681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:54.105681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:54.105681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:54.105681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:54.105681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:54.105681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:54.105681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:54.105681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:54.105681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:54.105681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:54.105681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:54.107301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce8f16e3-ef2d-46ef-9259-8b27100c0984]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:54.108483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:54.111468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:54.115114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25cf40fd-2f58-4a90-bfcc-37290c9db4c8]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.115932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:54.116252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:54.116562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:54.116948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:54.119471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[afa0ef12-ce45-4a66-b2cd-65f95fa910a5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.120503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c7a0699-4dc1-4e8d-af14-d395b26eed1d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.121398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5474e68b-b6d9-4dd3-9301-aa9b6faa2bcb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.143277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a869290-282d-4171-a08a-adcca3c26d60]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.161656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96fa3a58-046e-429d-9139-54b88124adf8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 274997, 'reachable_time': 39416, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 136925, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.164224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:54.164474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d3e8fc4-c6ee-479a-b626-3b91a20a602c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:54.166296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:51Z", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:54.168508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 5047d151-2cc9-4333-ae13-1bfa2875eddf. Wait time elapsed: 44.691 Jul 01 09:30:54.181745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 5047d151-2cc9-4333-ae13-1bfa2875eddf. Wait time elapsed: 44.691 Jul 01 09:30:54.181745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:54.194493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[632d3ae8-6d0d-41a6-a058-43cba936b30e]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:30:54 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:30:54 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:30:54 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:30:54 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:30:54 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:30:54 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:30:54 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:30:54 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.200825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.240s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:54.201372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac was not found. Performing router cleanup Jul 01 09:30:54.202615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bf5079a-9c6c-432d-ab8b-a62608cf2c34]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.203173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:54.203498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:54.203753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, update_id b7b427a9-1603-406b-a30c-bc4835cf0115. Time elapsed: 0.244 Jul 01 09:30:54.241423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.766s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:54.241839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:30:54.243328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e14d7e7e-12a7-4705-8830-78f188a4bb13]: (4, ['qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.243860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:54.244142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:54.244381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id f3ea2fc4-aca5-4f97-8d17-e60bb717a165. Time elapsed: 0.770 Jul 01 09:30:54.274002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44f5c766-8e58-4c44-8efb-44fb82a226b4]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.274482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:30:54.274793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:30:54.275052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:30:54.275334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4da8db80-0643-4392-ad3e-03f5c3888347" released by "apply_metadata_nat_rules" :: held 0.342s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:30:54.275849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-4da8db80-0643-4392-ad3e-03f5c3888347 Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.haproxy Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 4da8db80-0643-4392-ad3e-03f5c3888347 Jul 01 09:30:54.276280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:30:54.280442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/4da8db80-0643-4392-ad3e-03f5c3888347.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:30:54.313498 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-4da8db80-0643-4392-ad3e-03f5c3888347[136929]: Proxy listener started. Jul 01 09:30:54.316149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 4da8db80-0643-4392-ad3e-03f5c3888347 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:30:54.316660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 4da8db80-0643-4392-ad3e-03f5c3888347, update_id ab0dd2f7-7262-4b4d-b5f4-cff3fb7e004d. Time elapsed: 3.798 Jul 01 09:30:54.316937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4da8db80-0643-4392-ad3e-03f5c3888347, action 1, priority 1, update_id b3dd4d2d-2e6f-4a38-9d00-d2d2c067c413. Wait time elapsed: 1.307 Jul 01 09:30:54.317184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4da8db80-0643-4392-ad3e-03f5c3888347, action 1, priority 1, update_id b3dd4d2d-2e6f-4a38-9d00-d2d2c067c413. Wait time elapsed: 1.307 Jul 01 09:30:54.317477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 4da8db80-0643-4392-ad3e-03f5c3888347 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:30:54.318781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:54.339236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fd41e5f-04ea-41c9-a4ff-2cd48d09c813]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.341349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d15a8762-8213-40b0-9068-baa376c12e0b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.342760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5b9241b-5967-46c2-ad7d-132f7447108e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.343409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:30:54.344704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b712245b-62ba-47f7-9694-5edd7b907f6e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.374131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8c3f941-a170-41c1-aa45-4fca71c0f172]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 274918, 'reachable_time': 18968, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 136935, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.375304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:54.375800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:54.395846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad5cc8cb-8f41-48c5-b0c9-5fba5147f471]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 274918, 'reachable_time': 18968, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 136936, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.397212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:30:54.397329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:54.397606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4da8db80-0643-4392-ad3e-03f5c3888347.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:54.397858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:54.398602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 4da8db80-0643-4392-ad3e-03f5c3888347 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:30:54.399704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d38abbea-c67e-4518-95ec-bd3a39bb6774]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.440586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03fa07d5-5b4e-4b7f-a833-68ac6182c4db]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 274918, 'reachable_time': 18968, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 136937, 'error': None, 'target': 'qrouter-4da8db80-0643-4392-ad3e-03f5c3888347', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.443961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4da8db80-0643-4392-ad3e-03f5c3888347 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:54.444434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca711974-ebe7-4ef7-a31a-f771f9437252]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:54.444964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 4da8db80-0643-4392-ad3e-03f5c3888347 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:30:54.445475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (4da8db80-0643-4392-ad3e-03f5c3888347), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:54.446207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:54.446624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:54.447339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:54.447764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:54.448150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 4da8db80-0643-4392-ad3e-03f5c3888347, update_id b3dd4d2d-2e6f-4a38-9d00-d2d2c067c413. Time elapsed: 0.131 Jul 01 09:30:55.084249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:30:55.472324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b4ef55db-7211-4a65-b7cd-a85f39138d25 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['9b96a59e-ed9b-4059-a7ba-61d79a76a3ac'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:55.474840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, action 3, priority 1, update_id b8408798-b740-4776-ac48-0d81b578879b. Wait time elapsed: 0.002 Jul 01 09:30:55.475312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, action 3, priority 1, update_id b8408798-b740-4776-ac48-0d81b578879b. Wait time elapsed: 0.002 Jul 01 09:30:55.475868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:55.619252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.143s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:55.619723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac was not found. Performing router cleanup Jul 01 09:30:55.621907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9329b100-8337-4357-8eb4-29c22b37b1ae]: (4, ['qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:55.622862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:55.623310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:55.623839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, update_id b8408798-b740-4776-ac48-0d81b578879b. Time elapsed: 0.149 Jul 01 09:30:57.542957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-cb87fe2a-1f92-4532-819d-7248c4bb95c4 tempest-TestNetworkAdvancedServerOps-636282898 tempest-TestNetworkAdvancedServerOps-636282898-project] Got routers updated notification :['58317645-6e22-4a4b-8851-15323d1dba25'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:57.780345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8cdd68bd-ca02-49db-95ba-86d4a31600d5 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['e1dc75ae-b350-4801-91dc-89215c41ca50'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:57.783568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 1, update_id 4e8d824c-7889-4025-984c-9cda79653220. Wait time elapsed: 0.002 Jul 01 09:30:57.784342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 1, update_id 4e8d824c-7889-4025-984c-9cda79653220. Wait time elapsed: 0.003 Jul 01 09:30:57.785132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:58.247683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-96dbd5c3-a9b5-4a9a-9a55-d133f00e4d33 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['34501603-2ac8-4862-9890-211ee16385e3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:30:58.248878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 34501603-2ac8-4862-9890-211ee16385e3, action 3, priority 1, update_id ff5f1f46-2bd3-4190-b7ef-344349f79c3f. Wait time elapsed: 0.001 Jul 01 09:30:58.249482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 34501603-2ac8-4862-9890-211ee16385e3, action 3, priority 1, update_id ff5f1f46-2bd3-4190-b7ef-344349f79c3f. Wait time elapsed: 0.001 Jul 01 09:30:58.250202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:30:58.508378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.258s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:58.509534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 34501603-2ac8-4862-9890-211ee16385e3 was not found. Performing router cleanup Jul 01 09:30:58.513307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9ce581e-9695-4020-bcca-cd3dc7d94f7c]: (4, ['qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:58.514337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:58.514746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:58.515640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 34501603-2ac8-4862-9890-211ee16385e3, update_id ff5f1f46-2bd3-4190-b7ef-344349f79c3f. Time elapsed: 0.266 Jul 01 09:30:58.574841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.407s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:30:58.576036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:30:58.576882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:30:58.577199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:58.577583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:58.578885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c0cac97-5168-4761-8199-86154faf72a1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:58.613160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a4476b0-ce48-4a19-b98a-a7f5bbd76ee5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:58.649869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a26a2ac-aeb4-49d4-9f2a-85742325c97f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:58.730382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f0b8232-bbfd-4499-aec4-eb9a5a53862b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137008, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:58.767678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[467b834e-d037-47b4-858a-cfd0bec884ac]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:58.819955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a1ca00e-30fe-4a5c-b74d-8b50830a3dcf]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:58.859029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a518c871-ddcb-42ab-b807-a6ed5c6b7c09]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:58.893037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2eb8d00f-b7e5-4e3c-af59-85bdf3528ca5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:58.894229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:30:58.894506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:30:58.894771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:30:58.896308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:30:58.929371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:30:58.930169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad412d5f-51e2-4977-830a-3efc3a5b5124]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:58.933839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab7bc69e-fd49-4610-823f-7e5e165cc91a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:58.935931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:58.979976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:58.981521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:58.982326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:58.982969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:58.983552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:30:59.037260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a8fe394-8a7a-4d64-8ff2-86fa5fcc9f5d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:59.098618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6df84e4b-6004-442b-89f5-d55afa885197]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:30:59.098618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:30:59.098618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:30:59.098618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:30:59.098618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:30:59.098618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:30:59.098618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:30:59.098618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:30:59.098618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:30:59.098618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:30:59.098618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:30:59.098618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:30:59.098618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:59.099967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6df84e4b-6004-442b-89f5-d55afa885197]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:59.101055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:59.104053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:59.108377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7f18cd6-e277-4528-869e-ed98efeebd2c]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:59.110479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:59.110956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:59.111355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:30:59.112540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:30:59.114421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6db9780e-ba84-4ef3-b4e9-ae7a465aaf0f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:59.116866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c305705a-6476-4790-823f-0f9da4430618]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:59.118228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20ae6b36-25d6-407e-9438-e9e1a8b81590]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:59.155922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[409b322a-f74d-4b08-96a2-8976400d1de8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 275499, 'reachable_time': 29316, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137030, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:59.159256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:30:59.159701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4963b2d-dfda-4643-bbcc-a5b58d9f90ee]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:30:59.161585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2c5c1a36-bdc1-4847-9f8f-e598677fd30a", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.206", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "5cd44a2d-bdd8-4364-8080-ef931dc139c1", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1347, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dd:94:6c", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0e6c39b8-7e9c-439c-86ab-2385f592e29b", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:53Z", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:53Z", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:30:59.166246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:53Z", Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:30:59.178782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:30:59.182046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 1, update_id 4e9c69aa-93d9-43ae-87c3-da0f4ae9620e. Wait time elapsed: 1.633 Jul 01 09:30:59.182046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 1, update_id 4e9c69aa-93d9-43ae-87c3-da0f4ae9620e. Wait time elapsed: 1.636 Jul 01 09:30:59.182571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:00.244632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-0bfd0a76-44db-4b8a-8fd8-2c608641f127 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['34501603-2ac8-4862-9890-211ee16385e3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:00.246379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 34501603-2ac8-4862-9890-211ee16385e3, action 3, priority 1, update_id 9e251b15-1de7-4236-b98e-89ea18d88884. Wait time elapsed: 0.001 Jul 01 09:31:00.246812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 34501603-2ac8-4862-9890-211ee16385e3, action 3, priority 1, update_id 9e251b15-1de7-4236-b98e-89ea18d88884. Wait time elapsed: 0.001 Jul 01 09:31:00.248143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:00.358280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.110s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:00.358540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 34501603-2ac8-4862-9890-211ee16385e3 was not found. Performing router cleanup Jul 01 09:31:00.359835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50f25a0b-e843-443f-ae05-109075bc847d]: (4, ['qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:00.361189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:00.361829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:00.362336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 34501603-2ac8-4862-9890-211ee16385e3, update_id 9e251b15-1de7-4236-b98e-89ea18d88884. Time elapsed: 0.116 Jul 01 09:31:00.797676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-6b881a9b-950e-4c6b-8ee9-d48557623487 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['9b96a59e-ed9b-4059-a7ba-61d79a76a3ac'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:00.798902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, action 3, priority 1, update_id 33f3e61e-63b8-45a7-a413-ccb5a9d49d7b. Wait time elapsed: 0.001 Jul 01 09:31:00.799387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, action 3, priority 1, update_id 33f3e61e-63b8-45a7-a413-ccb5a9d49d7b. Wait time elapsed: 0.001 Jul 01 09:31:00.799893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:01.470089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c6b45fcd-0927-419f-84c7-5cfd2f0da534 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got router deleted notification for 34501603-2ac8-4862-9890-211ee16385e3 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:31:01.472375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 34501603-2ac8-4862-9890-211ee16385e3, action 1, priority 1, update_id 418fb5ef-4a0f-4038-b45f-87e86f80ef32. Wait time elapsed: 0.001 Jul 01 09:31:01.472879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 34501603-2ac8-4862-9890-211ee16385e3, action 1, priority 1, update_id 418fb5ef-4a0f-4038-b45f-87e86f80ef32. Wait time elapsed: 0.002 Jul 01 09:31:01.473436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 34501603-2ac8-4862-9890-211ee16385e3 was not found. Performing router cleanup Jul 01 09:31:01.476789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ae9a686-2330-4e4c-a457-ad3bc8a3621b]: (4, ['qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:01.477589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:01.478239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:01.478721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 34501603-2ac8-4862-9890-211ee16385e3, update_id 418fb5ef-4a0f-4038-b45f-87e86f80ef32. Time elapsed: 0.006 Jul 01 09:31:01.497934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.698s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:01.500118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac was not found. Performing router cleanup Jul 01 09:31:01.502777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31e291fa-1b80-4e7e-a97b-7d2668e8bff1]: (4, ['qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:01.504906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:01.505541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:01.506028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, update_id 33f3e61e-63b8-45a7-a413-ccb5a9d49d7b. Time elapsed: 0.707 Jul 01 09:31:02.667232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.882s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:02.668248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:02.670373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:02.670933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:02.671452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:02.673833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9fdce715-bab0-4958-a7d6-3756eb25ae82]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.701660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da599551-2743-46c1-8906-41f747e82061]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.707173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.525s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:02.708773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:02.710850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:02.711424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:02.712152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:02.719428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a21ea50-07e3-46aa-8215-cc4dc400f3c5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.748021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31bed1b9-e157-49ee-a378-10a156afd872]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.757721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ced6d4b3-8983-4a3e-af57-f21bd3af330b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.776838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a07a4bb-c6b3-4231-b096-cfed5eba68d0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.790434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-e7af5d45-2ce0-4bfc-bd5e-ff13f3e5219d tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got router deleted notification for 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:31:02.792259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, action 1, priority 1, update_id 5064f0e6-93a6-470c-8a67-53e4e846af22. Wait time elapsed: 0.001 Jul 01 09:31:02.793160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, action 1, priority 1, update_id 5064f0e6-93a6-470c-8a67-53e4e846af22. Wait time elapsed: 0.002 Jul 01 09:31:02.793511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac was not found. Performing router cleanup Jul 01 09:31:02.798579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f96a7719-2d7a-4466-90bc-5dc4325dd7fb]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.806178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:02.806809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.007s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:02.807308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9b96a59e-ed9b-4059-a7ba-61d79a76a3ac, update_id 5064f0e6-93a6-470c-8a67-53e4e846af22. Time elapsed: 0.016 Jul 01 09:31:02.815756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8af0b723-daaa-46f6-bca2-5945c168caea]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137038, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.833000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24cbb45f-3f54-4ab0-a42b-f350c195559a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137040, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.853129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28e726cf-b38d-4a38-a53c-386424df2407]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.862080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ea111b8-a4a0-434e-97c1-54558d6fbf17]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.881738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5dcef8a-25be-49c2-91f4-98a1efe3312c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.890640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d306253-4977-4fc8-bb8e-17661f818eb6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.910466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74e371f0-b392-4870-b2bc-75a2ecaedb1a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.922231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c4f5e6f-cdfd-48fb-b779-39c44494ee54]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.957155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2548f6df-28bb-4077-8a38-c1bdfdf911a1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.958109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:02.958419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:02.958687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:02.958972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:02.970086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c569c4e-c589-448b-a473-4d06e0ad5a73]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.970694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:02.971019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:02.971309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:02.971602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:02.994090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:02.994438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4947172-a0e8-4176-bd29-eb91e90f31f2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.994730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:02.995012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cdebed09-5e99-44ab-88d2-9c93155755e9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:02.999262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c61b46a0-6379-44d4-8902-ead0f7cc93af]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.000259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:03.002234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b7d3474-b647-4230-9fd9-1513b99f710c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.040040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:03.114160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:03.115095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:03.116064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:03.116592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:03.118368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:03.163187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:03.163896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:03.164287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:03.164673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:03.165102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:03.191534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[859fa73c-1920-49d9-8943-ff2a0636ca46]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.245946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[34a3df50-90ce-4e8b-addd-f40a7e5ae96a]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:03.245946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:03.245946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:03.245946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:03.245946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:03.245946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:03.245946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:03.245946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:03.245946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:03.245946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:03.245946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:03.245946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:03.245946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:03.246758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34a3df50-90ce-4e8b-addd-f40a7e5ae96a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.246758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa38eb79-99a5-4eae-b522-ee68a4ec9d10]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:03.248174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:03.249946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:03.251773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7104c973-84c1-4634-982a-614a82925599]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.251773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:03.251773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:03.251994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:03.252455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:03.253918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb8163f3-0fff-4a1a-bc80-2bd5bbdec408]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.255098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8cdee01c-958a-4c77-a92c-f55a1661b07d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.256944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63c4c473-4d42-46e6-aeac-8d545db4ec4e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8f661014-c026-4864-99bf-ddf676bb7a50]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:03.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:03.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:03.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:03.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:03.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:03.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:03.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:03.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:03.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:03.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:03.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:03.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:03.306262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f661014-c026-4864-99bf-ddf676bb7a50]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.307052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b069d51f-3e22-4544-bdf8-29dd67205b02]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 275907, 'reachable_time': 23952, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137077, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:03.310372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:03.317630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:03.323640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65e35d05-6acf-435a-86a9-276041cedd28]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.326150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:03.326688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2eb8ca49-a4a6-400f-9603-c786e4bd0366]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.329563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:03.331848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:03.332248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:03.333967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:03.335951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1c2933de-fbc4-4252-b66f-8f745f6b03a3", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.74", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4adaa360-ab59-46f0-a744-2c71b0db08dd", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1350, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:94:b6:4c", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "601e18e3-d27b-4d21-a5c3-a4d7827fe67d", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:56Z", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:56Z", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:03.341733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:50Z", Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:03.353262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e22e346b-ff12-4a56-a7c3-806f94069eeb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.359523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id 4e8d824c-7889-4025-984c-9cda79653220. Wait time elapsed: 5.570 Jul 01 09:31:03.359925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id 4e8d824c-7889-4025-984c-9cda79653220. Wait time elapsed: 5.579 Jul 01 09:31:03.360307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:03.387015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4493e874-6b73-4cb4-8a7f-b1aeac95d8b4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.392379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef1c2aaa-2cd6-4154-8d04-61708ebd884d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.431281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca6eb591-d804-4265-901b-9a317548c6f7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 275909, 'reachable_time': 42752, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137095, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.450480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:03.450697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86a81608-0a97-4a28-bb0c-e0c54f3e3fa2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:03.455376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2c5c1a36-bdc1-4847-9f8f-e598677fd30a", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.206", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "5cd44a2d-bdd8-4364-8080-ef931dc139c1", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1347, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dd:94:6c", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0e6c39b8-7e9c-439c-86ab-2385f592e29b", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:53Z", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:53Z", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:58Z", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:03.460425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:03.481216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 7b94701f-1773-4678-a963-a5ac13db9ced. Wait time elapsed: 43.429 Jul 01 09:31:03.481216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 7b94701f-1773-4678-a963-a5ac13db9ced. Wait time elapsed: 43.430 Jul 01 09:31:03.481216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:04.693249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-af974fd4-3db2-46d3-a431-78232c071b5f admin admin] Got routers updated notification :['22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:04.694542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 1, update_id 79a52ab3-eb65-4816-b403-5c714d0468f7. Wait time elapsed: 0.001 Jul 01 09:31:04.694806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 1, update_id 79a52ab3-eb65-4816-b403-5c714d0468f7. Wait time elapsed: 0.001 Jul 01 09:31:04.695101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:05.038193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.678s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:05.039084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:05.040115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:05.041923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:05.042415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:05.043808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d3c5f78-cd94-4170-8df5-cf5f8c4101af]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.081687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[889f5bc6-e069-4fdb-8c3a-be2d4b19088e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.114495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7dbface-c03e-4f90-ab22-61b7e3345710]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.173593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b5ad7c3-a7b2-4c15-b3c2-eae717fd7dce]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137156, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.210682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2efd3d2-ad60-445f-91e9-e5b8e1a8380f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.245390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[927cbbdf-3d91-4cbc-b9b6-e9546da5c95a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.275870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[565610ea-21c8-4372-a7fb-1679c9150c90]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.317278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dabdd55b-999a-40d0-8e70-28802fcb677f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.318480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:05.318959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:05.319497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:05.320097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:05.348742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:05.349377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[945e2718-3524-470e-b52c-e68752185df2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.352358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5173c68-0fd6-43cb-b7e3-1a358996cd2e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.353574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:05.397596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:05.398768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:05.399943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:05.400811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:05.401437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:05.463131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2beaa7c-f949-4678-a166-2f8cd52c31fc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.529156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[715ae3f7-21c5-4520-9124-6dd3def9083e]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:05.529156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:05.529156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:05.529156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:05.529156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:05.529156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:05.529156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:05.529156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:05.529156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:05.529156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:05.529156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:05.529156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:05.529156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:05.530543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[715ae3f7-21c5-4520-9124-6dd3def9083e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:05.532170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:05.533543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:05.535206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb9b171b-1764-4d82-8ddd-7d7ccfc8d3c4]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.535206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:05.535376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:05.535664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:05.536150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:05.539227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2866cc26-c2ed-4f52-8aa0-91ca3c0a9ffe]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.540829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7d732a3-42cc-491a-a10c-898f9af3e823]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.542225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0c3e93a-dfed-481c-9a38-cd98ac6c4931]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.583446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d463303b-fba6-4f38-9cc0-89ece07263d9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 276143, 'reachable_time': 41944, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137174, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.594261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:05.594744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d972ed60-18b0-4158-abfd-02415eceb1fa]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:05.595962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1c2933de-fbc4-4252-b66f-8f745f6b03a3", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.74", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4adaa360-ab59-46f0-a744-2c71b0db08dd", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1350, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:94:b6:4c", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "601e18e3-d27b-4d21-a5c3-a4d7827fe67d", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:56Z", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:56Z", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:05.599747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:03Z", Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id 4e8d824c-7889-4025-984c-9cda79653220. Wait time elapsed: 7.833 Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id 4e8d824c-7889-4025-984c-9cda79653220. Wait time elapsed: 7.834 Jul 01 09:31:05.617200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:06.444810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-61b34a31-44de-4d69-9d7a-4ae796e4ebd1 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['3f0d0f86-8039-49c8-9d77-e288eebb1d4f'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:06.445865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 3f0d0f86-8039-49c8-9d77-e288eebb1d4f, action 3, priority 1, update_id 881fb9f8-c80d-4422-8cf7-f503e94135fd. Wait time elapsed: 0.001 Jul 01 09:31:06.446215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 3f0d0f86-8039-49c8-9d77-e288eebb1d4f, action 3, priority 1, update_id 881fb9f8-c80d-4422-8cf7-f503e94135fd. Wait time elapsed: 0.001 Jul 01 09:31:06.446605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:06.473248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.020s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:06.474034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:06.475157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:06.475708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:06.476205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:06.477584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62d4b1f5-b2d4-4b9d-bce3-4088aec3ef4b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.504979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4095b2d-d783-44a9-8595-064a33223a93]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.557642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e13dfc8d-4821-47ad-9a9a-978be79a3b35]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.616165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a77b9c9-99d1-4eec-9d7d-8f3bb5d4f41a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137177, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.661023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c794daea-eb35-466f-9afe-ca419bd293ee]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.696668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48cc0ae2-50e3-437b-9d03-b121648c0759]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.740670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fd33290-6be8-493a-9721-c9db3ea20c8b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.775959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.329s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:06.776266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 3f0d0f86-8039-49c8-9d77-e288eebb1d4f was not found. Performing router cleanup Jul 01 09:31:06.778676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c16fe160-389c-400e-80b1-4189bb288af3]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.779424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:06.779863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:06.780260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 3f0d0f86-8039-49c8-9d77-e288eebb1d4f, update_id 881fb9f8-c80d-4422-8cf7-f503e94135fd. Time elapsed: 0.334 Jul 01 09:31:06.781110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90667c2f-b1ed-4efc-9edb-597b70893f04]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.781823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:06.782231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:06.782633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:06.783043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:06.815315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:06.816648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81b42e42-284d-4623-b500-c1155d4b1fc5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.822048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[583e3b0a-399a-4eff-87e6-558bfefb10c6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.824750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:06.860986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.166s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:06.861463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:06.863345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:06.864051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:06.864479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:06.867454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4cf24cb-d929-4b97-89d3-8ef0be07a0f6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.868212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:06.869299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:06.870407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:06.870977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:06.871535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:06.917843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34e1f4e5-8374-44dd-99ed-237f5c88fd22]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.945455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6d0fae9-2ef5-4982-98ae-66a8c4009cef]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.975775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55e25a29-4586-491e-a477-e194c13c2477]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.976926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[01478f8c-9f0a-4fde-bf5b-4ff2e3dc548e]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:06.976926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:06.976926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:06.976926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:06.976926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:06.976926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:06.976926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:06.976926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:06.976926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:06.976926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:06.976926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:06.976926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:06.976926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:06.979411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01478f8c-9f0a-4fde-bf5b-4ff2e3dc548e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:06.986489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:06.989679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:06.993406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec19d96a-bccc-4f7f-b783-ee56472a697b]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:06.994346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:06.994803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:06.995198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:06.995693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:06.998881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[804ca0ed-b9a3-417c-ba88-2c37756282d3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.000136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d34c842-b215-4178-af57-0198bb093286]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.001269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ce57f2d-d8c5-4c7a-b506-0d9281ba3d96]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.028929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bcec4a1-2db6-4277-9ff2-40ab415a814f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137198, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.044767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[823c9575-1c80-40a7-a51d-498f677bdb39]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 276287, 'reachable_time': 36408, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137199, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.047179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:07.047664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b3bf696-413d-48ec-b1e8-f139d799d1cc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:07.048791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2c5c1a36-bdc1-4847-9f8f-e598677fd30a", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.206", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "5cd44a2d-bdd8-4364-8080-ef931dc139c1", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1347, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dd:94:6c", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0e6c39b8-7e9c-439c-86ab-2385f592e29b", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:53Z", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:53Z", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:07.053174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:02Z", Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 5047d151-2cc9-4333-ae13-1bfa2875eddf. Wait time elapsed: 57.586 Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 5047d151-2cc9-4333-ae13-1bfa2875eddf. Wait time elapsed: 57.586 Jul 01 09:31:07.066308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:07.090583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[442ed886-0f7e-4550-90b5-8b4cb26c5040]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.125683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bde2d286-9713-4a97-af70-2beb952787d6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.147674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[597ce866-f630-4faf-8944-f3442714a2da]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.171657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7c805a4-09a3-4d2c-8de8-88eb026c7319]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.171657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:07.171657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:07.171657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:07.200940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a451bb90-e563-4b1c-b06a-b6ee69599b20]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 276329, 'reachable_time': 20408, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137205, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.202148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:07.202613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:07.203262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:07.203815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:31:07.204316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:07.204588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:07.204873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'b09b9e0f-29ce-44ec-926d-d74f2c05ee7a', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:2c:9f:1d', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'device_owner': 'network:router_gateway', 'standard_attr_id': 1367, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.57', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:31:01Z', 'updated_at': '2022-07-01T09:31:06Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-b09b9e0f-29), ns(qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:31:07.230197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-b09b9e0f-29 not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:07.230556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f20f5dca-6f47-41bd-bcd0-41db010f7264]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.234320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45d26fed-09bb-4609-a8af-a0a01c380c9c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.235378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-b09b9e0f-29, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:07.235850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:31:07.236598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-b09b9e0f-29, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:07.237261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-b09b9e0f-29, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:07.237647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-b09b9e0f-29, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:07.238022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-b09b9e0f-29, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:07.238397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-b09b9e0f-29, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b09b9e0f-29ce-44ec-926d-d74f2c05ee7a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:2c:9f:1d'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:07.280232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22273e35-94d1-4f39-aeb8-57e2e6d7b399]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.324881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[14aa3f2a-3222-4a33-9812-e96e923be78a]: Network interface qg-b09b9e0f-29 not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:07.324881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:07.324881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:07.324881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:07.324881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:07.324881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:07.324881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:07.324881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:07.324881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:07.324881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:07.324881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:07.324881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:07.324881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b09b9e0f-29 not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:07.326303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14aa3f2a-3222-4a33-9812-e96e923be78a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-b09b9e0f-29 not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.326979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:07.327331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:31:07.349602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b02039dc-8b87-48c6-bc83-2c8c30cbe204]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.546236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d13c098c-48a6-430a-b696-1a56d030faa1]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.592202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[540f5b5f-6405-46dd-88ed-89566948e729]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.794819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10acda9c-df3a-4a0a-8440-e434d7ce4caa]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.796858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:31:07.799012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:07.799706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:07.800318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" released by "process_external" :: held 0.597s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-b09b9e0f-29 not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b09b9e0f-29 not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b09b9e0f-29 not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:07.803394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b09b9e0f-29 not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b09b9e0f-29 not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:07.808734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:07.814848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[551a12ff-ea5a-4b54-8fd9-0fc76487cd4a]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.819152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:07.819745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:07.820165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:07.821253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:07.823136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06a9f2d5-c5cf-42ac-8eee-54d8c052f76c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.826263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d83d357-4fb4-4cb5-94a2-033f4cdeb3d8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.832250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08e722c8-4a5b-41c7-b9e4-d6d3c395d99c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.868263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e120a02-e75b-4562-8059-ab0b01423b63]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 276329, 'reachable_time': 20408, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137222, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.870873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:07.871408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e492b5f-c0c8-4c95-896b-b92f2a22cf9c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b09b9e0f-29 not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b09b9e0f-29 not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:07.877610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:02Z", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:06Z", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id 79a52ab3-eb65-4816-b403-5c714d0468f7. Wait time elapsed: 3.185 Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id 79a52ab3-eb65-4816-b403-5c714d0468f7. Wait time elapsed: 3.185 Jul 01 09:31:07.881187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:08.895464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c7bec559-6145-4ba3-8de2-d8cc0ba1c642 admin admin] Got routers updated notification :['22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:10.101363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.487s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:10.101949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:10.107117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:10.107767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:10.108915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:10.111000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c2befea-4364-45c3-acdc-e61d9bb336f7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.156123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01ffb38e-b22e-46b0-92ac-1ef6989779a1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.205102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63852360-d4f3-461d-ac82-29ac617b88f3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.254749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a088bea-d578-4b98-97e6-2cc42ca1e88b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137225, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.279144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b91ca15b-5904-4d17-99b2-3c556afddd3b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.313895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74d1d774-f278-42d1-8544-f206d1a403e2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.374931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[358e96e9-c815-4e35-994a-512a6018e0e7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.409145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93ed261e-dbf4-4934-ba82-4d708a8299af]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.413126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:10.413592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:10.414532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:10.414986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:10.452865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:10.452865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17d2b56f-839e-458c-9f31-4b6a38c27032]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.456427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db28935d-a70c-477d-84fe-02c776c91729]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.458982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:10.501309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:10.504894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:10.505447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:10.505976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:10.506902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:10.564773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af41574c-5c6a-4275-abd0-558fec6184d4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.606734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b10a748d-59b0-4bbd-ae8e-5b50f0f4deaa]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:10.606734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:10.606734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:10.606734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:10.606734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:10.606734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:10.606734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:10.606734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:10.606734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:10.606734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:10.606734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:10.606734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:10.606734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:10.609696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b10a748d-59b0-4bbd-ae8e-5b50f0f4deaa]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:10.612329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:10.617180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:10.621175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bdb689b4-5e70-42dc-b38b-ed93798095bf]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.622360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:10.623110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:10.623622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:10.624275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:10.625664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d61b3529-0481-4c55-85fb-e86b9eb62ed5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.633307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6c619fa-119d-42c6-81e0-4f41c8b07301]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.634568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b1c16d7-3c10-4da4-888a-e815baebfbec]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.678936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c56e0157-0c0e-4fc7-ad63-ae06877709e5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 276651, 'reachable_time': 31656, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137244, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.682088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:10.682602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[396f8026-9375-4796-a2e9-7cdaf4df32df]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:10.690090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1c2933de-fbc4-4252-b66f-8f745f6b03a3", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.74", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4adaa360-ab59-46f0-a744-2c71b0db08dd", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1350, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:94:b6:4c", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "601e18e3-d27b-4d21-a5c3-a4d7827fe67d", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:56Z", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:56Z", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:10.696020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:05Z", Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 13, Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:10.712370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id 4e8d824c-7889-4025-984c-9cda79653220. Wait time elapsed: 12.930 Jul 01 09:31:10.717443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id 4e8d824c-7889-4025-984c-9cda79653220. Wait time elapsed: 12.931 Jul 01 09:31:10.717968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:11.524664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.645s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:11.525611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:11.526820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:11.527617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:11.528645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:11.529991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ab2d187-61e3-478a-9c94-c14bb94a831a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.542682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ec31d9c-f28c-4d8b-9bb7-d11f1a5ea2fa]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.582064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f214b62a-1fe7-4025-bec0-eeeced394abc]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.633775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1847b18a-e6de-48be-9588-1993b42937ec]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137300, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.709471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-fabafd88-9db9-4758-b4e4-c86a026c452d tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got routers updated notification :['f6699386-131a-4161-86e3-afc4a0d261a4'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:11.710280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4c78f06-5059-4bf7-ba0e-06c14696de16]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.712144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 4768d1c7-d523-452b-8556-c7f8affabaec. Wait time elapsed: 0.002 Jul 01 09:31:11.712597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 3, priority 1, update_id 4768d1c7-d523-452b-8556-c7f8affabaec. Wait time elapsed: 0.003 Jul 01 09:31:11.713089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:11.718441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-411b2d54-6414-45c3-b8c0-73de5182a7a7 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['3f0d0f86-8039-49c8-9d77-e288eebb1d4f'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:11.721443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 3f0d0f86-8039-49c8-9d77-e288eebb1d4f, action 3, priority 1, update_id c2d19560-ca2a-41c8-a1ad-34319bad0506. Wait time elapsed: 0.002 Jul 01 09:31:11.721936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 3f0d0f86-8039-49c8-9d77-e288eebb1d4f, action 3, priority 1, update_id c2d19560-ca2a-41c8-a1ad-34319bad0506. Wait time elapsed: 0.003 Jul 01 09:31:11.722442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:11.753989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b11a2f0-4ee7-461b-8916-d8ef46dbae3a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.790453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e216b58d-a9a0-454f-9beb-b8f0028ac2d3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.826869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d0917b2-35bb-49a1-8a41-24afd5fe46f3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.827744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:11.828431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:11.831081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:11.831604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:11.861110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:11.861820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[091bd4b6-637e-453d-94a4-26e87ab663a6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.865061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83834478-9016-4552-9a97-0102e4a602ae]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.867046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:11.867430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:31:11.868562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:11.869503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:11.870107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:11.870677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:11.871231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:11.915898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.203s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:11.916405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:31:11.917807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c13f9cd-ad1d-4c79-97cd-f2a73822e24e]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.920251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:11.920754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:11.921178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id 4768d1c7-d523-452b-8556-c7f8affabaec. Time elapsed: 0.209 Jul 01 09:31:11.933364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83e2d3d1-fdaf-40e1-bf24-b108aaec7761]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2f5a2b26-51a1-44d0-b13b-3ac4d8ff7922]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:11.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:11.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:11.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:11.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:11.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:11.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:11.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:11.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:11.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:11.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:11.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:11.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:11.970809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f5a2b26-51a1-44d0-b13b-3ac4d8ff7922]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:11.971200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:11.972561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:11.974306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89378b05-1be2-4905-8bbd-dbd48ae98028]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.974306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:11.974306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:11.974306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:11.975186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:11.975671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52f44288-7d33-4241-9c72-2cb03fddacf8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.976560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84dd6020-1983-4039-8627-5560e9a04fd7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.977497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42f49459-bf3d-40d2-be35-70120207d35f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:11.999427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a405274-6126-4108-a91f-635ac41de292]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 276789, 'reachable_time': 37136, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137329, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:12.001243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:12.001696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbcbbf80-1e7e-4b5b-9801-a5780c014cd3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:12.002826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:06Z", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:10Z", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:12.007354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:12.020506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 1, update_id c861a568-988e-40ae-a7bb-0a3190f37a5d. Wait time elapsed: 3.119 Jul 01 09:31:12.020506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 1, update_id c861a568-988e-40ae-a7bb-0a3190f37a5d. Wait time elapsed: 3.120 Jul 01 09:31:12.020506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:12.456151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c09979be-a563-4ee9-801e-5aac7fd5bc94 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got router deleted notification for 3f0d0f86-8039-49c8-9d77-e288eebb1d4f {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:31:12.797672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.075s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:12.798258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 3f0d0f86-8039-49c8-9d77-e288eebb1d4f was not found. Performing router cleanup Jul 01 09:31:12.798972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2235348e-5462-4c40-8adf-46baea09bbbe]: (4, ['qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:12.799870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:12.800389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:12.800845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 3f0d0f86-8039-49c8-9d77-e288eebb1d4f, update_id c2d19560-ca2a-41c8-a1ad-34319bad0506. Time elapsed: 1.080 Jul 01 09:31:12.801341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 3f0d0f86-8039-49c8-9d77-e288eebb1d4f, action 1, priority 1, update_id a99247dd-67c8-466a-9cca-bc63cc0d5072. Wait time elapsed: 0.345 Jul 01 09:31:12.803313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 3f0d0f86-8039-49c8-9d77-e288eebb1d4f, action 1, priority 1, update_id a99247dd-67c8-466a-9cca-bc63cc0d5072. Wait time elapsed: 0.345 Jul 01 09:31:12.803829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 3f0d0f86-8039-49c8-9d77-e288eebb1d4f was not found. Performing router cleanup Jul 01 09:31:12.805011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b6200e9-497f-4cb5-9ee0-3a1f5b74b269]: (4, ['qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:12.806160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:12.806629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:12.807073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 3f0d0f86-8039-49c8-9d77-e288eebb1d4f, update_id a99247dd-67c8-466a-9cca-bc63cc0d5072. Time elapsed: 0.006 Jul 01 09:31:12.913740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.897s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:12.916486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:12.917754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:12.918083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:12.919023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:12.923132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0fe03ea-1363-4468-a41e-fd5c2c1ae0a9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:12.941493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48ef37f5-1b6d-4843-aa16-9f1255c75040]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:12.963772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fd73e1c-d610-405c-be62-4df9d68fe497]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:12.994399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.932s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:12.995678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:12.996834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:12.997935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:12.998652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:12.999858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97680a2e-b8f1-4e20-9c8d-320a23fe33f5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137333, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.003995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37fa7b67-9a2d-45f0-bdc1-1d0cf9e879c5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.024524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7028dd28-93fd-4e9d-b0e7-465d8473be94]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.029800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c769a675-09a1-4d47-aaf3-939d0f8b217d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.057076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e36b3df-fbc6-47f4-ba6a-d554964b7348]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.057860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8702ac56-702f-490e-8972-31294a3a1a81]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.082980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e47c4a1c-4324-4508-8ea9-d8f11100d106]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.100157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0bd51d8a-e961-442b-831c-11fe5df10e21]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137340, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.109587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c505f7bc-014e-4900-9df6-c1a614a0c9e8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.111636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:13.112072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:13.112631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:13.113680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:13.121198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[430c72d9-c85f-497e-aceb-dc4056a92521]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.137773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:13.138428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[777b018d-d295-4313-92fe-563b2e5a16b9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.141785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8980bb30-14e6-4896-b811-ff854a154a0d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.143349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:13.161282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0312082a-696e-47e3-ad1c-25f356446680]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.179106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:13.181012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:13.181641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:13.182443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:13.183131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:13.239406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[785c503a-ac8c-4ac1-96a1-690ab66a651a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.246872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76efa6b6-6933-492e-b167-0e5e3244b49d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.265036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a416ea4b-f181-4886-b560-86603e9aac24]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.266016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:13.266505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:13.267218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:13.267880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:13.297771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:13.298370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c063af83-e970-45db-9276-a2ab53d1628b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.301724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1869de89-278b-4992-8149-05a8819f1cbb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.303546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:13.318341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1cc3f05e-4333-4f2a-ac03-1d23df74e5a1]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:13.318341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:13.318341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:13.318341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:13.318341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:13.318341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:13.318341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:13.318341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:13.318341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:13.318341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:13.318341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:13.318341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:13.318341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:13.319199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1cc3f05e-4333-4f2a-ac03-1d23df74e5a1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:13.320012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:13.321463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:13.327600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[924d63b2-e258-4749-96e7-17671921d245]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.327600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:13.327600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:13.328038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:13.328258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:13.329361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f5aaa3b-ee6e-4b70-89b3-c59eb998d8d8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.330519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1a2e678-e225-4e11-90fd-2aa68ed3220c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.341419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:13.342338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:13.342794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:13.343466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:13.343978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:13.354491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd17910a-ebc3-404b-8123-19b25b6eefc2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.405655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[389d8054-9f3d-4de9-927d-05fe916011d7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.426828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d12bd17-e87f-4196-a0aa-eab58b2601b2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 276925, 'reachable_time': 29012, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137363, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.435563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:13.435563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f10f5bd-063d-4f98-afa9-80cecfeb5fb7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:13.436643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:06Z", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:10Z", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:13.438885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id 79a52ab3-eb65-4816-b403-5c714d0468f7. Wait time elapsed: 8.744 Jul 01 09:31:13.449196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id 79a52ab3-eb65-4816-b403-5c714d0468f7. Wait time elapsed: 8.744 Jul 01 09:31:13.449196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:13.451518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2a0f8524-34d8-451f-a801-8ecae601c6d7]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:13.451518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:13.451518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:13.451518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:13.451518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:13.451518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:13.451518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:13.451518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:13.451518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:13.451518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:13.451518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:13.451518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:13.451518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a0f8524-34d8-451f-a801-8ecae601c6d7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:13.452941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:13.454914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:13.457974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f442828e-22ea-4f87-833c-a36729fff6ae]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.457974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:13.457974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:13.457974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:13.457974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:13.458684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e6d8410-b9b0-440c-a793-762b11b9e304]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.459912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6598e018-fbbe-4928-a8a9-5579004d99cd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.460880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[940235af-16e1-4d5b-8972-fe37592ecc5f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.482347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59d5423a-9226-49ee-9d82-17f5d7769ecb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 276936, 'reachable_time': 34860, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137372, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50b981f5-6be9-410a-8be6-685500e769e7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:13.486721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2c5c1a36-bdc1-4847-9f8f-e598677fd30a", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.206", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "5cd44a2d-bdd8-4364-8080-ef931dc139c1", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1347, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dd:94:6c", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0e6c39b8-7e9c-439c-86ab-2385f592e29b", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:53Z", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:53Z", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:08Z", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:13.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3 Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 4e9c69aa-93d9-43ae-87c3-da0f4ae9620e. Wait time elapsed: 15.943 Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 4e9c69aa-93d9-43ae-87c3-da0f4ae9620e. Wait time elapsed: 15.943 Jul 01 09:31:13.500972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:13.902110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-02a66dac-ef38-444d-963f-05b033b1ed11 tempest-FloatingIPTestJSON-1342949505 tempest-FloatingIPTestJSON-1342949505-project] Got router deleted notification for f6699386-131a-4161-86e3-afc4a0d261a4 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:31:13.903354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f6699386-131a-4161-86e3-afc4a0d261a4, action 1, priority 1, update_id f87a2f5c-e217-4e22-a3a8-4bd2c05acb65. Wait time elapsed: 0.001 Jul 01 09:31:13.904577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f6699386-131a-4161-86e3-afc4a0d261a4, action 1, priority 1, update_id f87a2f5c-e217-4e22-a3a8-4bd2c05acb65. Wait time elapsed: 0.001 Jul 01 09:31:13.905153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f6699386-131a-4161-86e3-afc4a0d261a4 was not found. Performing router cleanup Jul 01 09:31:13.906411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e029f524-68f7-45a3-8cd6-0e9cde614441]: (4, ['qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:13.907688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:13.908151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:13.909356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f6699386-131a-4161-86e3-afc4a0d261a4, update_id f87a2f5c-e217-4e22-a3a8-4bd2c05acb65. Time elapsed: 0.005 Jul 01 09:31:14.590589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.103s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:14.591019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:14.592104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:14.592566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:14.592987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:14.594925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e428479c-db9e-4b96-aaf3-0e89c353fb76]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:14.625181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[315fff44-4a65-46c3-9c33-c75957d9915c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:14.673575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70cf13a7-2f48-4163-a7f4-c581903ff3e6]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:14.717956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1c202ba-c706-4067-9779-f9d049d4e464]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137375, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:14.757873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01b4df75-48a4-410a-b5d2-632650fc7b22]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:14.780507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff5ab2e4-8d0a-4766-9d30-1b408e54cd8b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:14.813458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2a7bdb5-981e-4800-842e-3fe2287da073]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:14.836018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[008dca2f-0c68-4abe-851d-b7c9b2ed7b6c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:14.838142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:14.844675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:14.845329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:14.847122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:14.873337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:14.874830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23049049-5748-4a79-87f0-6315c35853fe]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:14.878259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[902923c0-daf3-4412-8650-92e4cdc6526d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:14.879552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:14.906605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:14.907350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:14.907833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:14.909659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:14.910188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:14.967748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4614718-3625-4f93-8f56-3379717a28d7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:15.026950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2314ddb2-676e-4adb-8a3d-fa254307b648]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:15.026950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:15.026950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:15.026950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:15.026950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:15.026950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:15.026950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:15.026950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:15.026950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:15.026950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:15.026950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:15.026950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:15.026950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:15.028324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2314ddb2-676e-4adb-8a3d-fa254307b648]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:15.029107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:15.031295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:15.033055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1cbaf98f-9ab3-4867-a9ce-b59b272ec9bb]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:15.033153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:15.033446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:15.033843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:15.034184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:15.035501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c6b0b6d-fd5e-40e0-99ff-3dfef433c7ab]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:15.036521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[872db340-91a4-4cc0-bd76-c4fa0e2ea5ee]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:15.038155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3251b24c-ab05-425e-9599-dc288514456e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:15.072678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13405689-6799-4804-ade7-40b5096bc47e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 277098, 'reachable_time': 44708, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137396, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:15.075229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:15.075845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4b58cbc-eb2a-4e19-a052-cdafd7a8ce71]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:15.078015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2c5c1a36-bdc1-4847-9f8f-e598677fd30a", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.206", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "5cd44a2d-bdd8-4364-8080-ef931dc139c1", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1347, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dd:94:6c", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0e6c39b8-7e9c-439c-86ab-2385f592e29b", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:53Z", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:53Z", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:15.082483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:08Z", Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:15.103338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:15.108829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 7b94701f-1773-4678-a963-a5ac13db9ced. Wait time elapsed: 55.082 Jul 01 09:31:15.110006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 7b94701f-1773-4678-a963-a5ac13db9ced. Wait time elapsed: 55.085 Jul 01 09:31:15.110006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:15.112329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:15.112687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:15.113119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:15.113403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:16.593037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.875s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:16.593965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:16.596909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:16.597379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:16.597800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:16.599394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fe60cad-d030-423c-a3fb-999b1ebc2c40]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:16.627464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[456f7de3-652a-4fec-a979-9c0dd77aa6b4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:16.677229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[856948d6-a65e-4ca8-b5c0-333f0e30454f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:16.731505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[011ef9ec-5b0e-4809-8b82-1d192bed2c96]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137399, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:16.757225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af9ac5b3-4e11-40fa-9c95-f206e7928a52]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:16.792964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf643378-4f9c-44a4-a90a-53e6e56e9d51]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:16.830649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd8286bf-2967-4e67-b496-d9174b7730f4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:16.857461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d504750-a58c-416f-9013-627f1f058864]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:16.858303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:16.858741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:16.859227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:16.859752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:16.890865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:16.891540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fc4eee8-bcc6-4ca0-8f3b-038d378d0a43]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:16.894402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea6fdc95-6506-46ff-a079-19111c6a913f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:16.895540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:16.945263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:16.945957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:16.946387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:16.946750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:16.948123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:17.002758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a83740ed-4a7f-456f-b21a-a9bffbfc0d31]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:17.046496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[fb1499e9-f655-45e0-8475-792dc42a6c8a]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:17.046496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:17.046496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:17.046496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:17.046496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:17.046496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:17.046496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:17.046496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:17.046496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:17.046496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:17.046496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:17.046496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:17.046496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:17.047413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb1499e9-f655-45e0-8475-792dc42a6c8a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:17.047657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:17.050660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:17.058576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f86e0fa-dec9-41dd-8b07-8cbb79d68df0]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:17.060932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:17.061367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:17.061619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:17.062061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:17.064418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9031bfe5-a63d-42ee-a675-713679bbb025]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:17.065449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2bd2b9c9-a089-4040-b900-d7ee7b61a436]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:17.066887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2da8cd78-216c-4da9-bb94-a1701c9da2e2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:17.109136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90ff8576-7128-48b6-9684-ebe5be7fd808]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 277299, 'reachable_time': 42152, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137418, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:17.111335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:17.111827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d08abf76-8f09-49f2-9b33-b9c10a2df39c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:17.113219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1c2933de-fbc4-4252-b66f-8f745f6b03a3", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.74", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4adaa360-ab59-46f0-a744-2c71b0db08dd", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1350, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:94:b6:4c", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "601e18e3-d27b-4d21-a5c3-a4d7827fe67d", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:56Z", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:56Z", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:17.115568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:09Z", Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id 4e8d824c-7889-4025-984c-9cda79653220. Wait time elapsed: 19.333 Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id 4e8d824c-7889-4025-984c-9cda79653220. Wait time elapsed: 19.333 Jul 01 09:31:17.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:18.058546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.620s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:18.059197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:18.060622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:18.060955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:18.061238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:18.062535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c34a820-dfd3-4d66-be24-8aa56cbed867]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.098142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6757c0c8-bd73-4eea-8eec-789f5a4bc4d8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.136731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[314cd954-3890-4110-a1c1-28dc8486ef9f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.190850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc3331c0-39a6-4a67-85dc-cdf894f403cc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137421, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.238104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c115b9f5-39ef-4cb5-8f17-00603f2d79e7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.275296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a7be83f-05bc-49e0-bca4-c5c607c5b8f7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.312246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1aef23fc-57a7-479c-bd04-8c76abb98499]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.326834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.212s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:18.327352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:18.328425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:18.328794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:18.329080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:18.331424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1dbed8f-d61c-4129-adae-c3e63e629538]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.359001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8281252-3d48-41f4-8281-af66b1579781]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.359939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:18.360401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:18.360802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:18.361269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:18.362483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60568ff1-8489-4122-90ff-875c7802cb58]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.414486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff7bde9e-779f-4fdf-ad30-8520f6cbd7e7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.415528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:18.416105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[afa72409-8dc8-4d2a-9136-4ae470c6058e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.424177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5677812-81c2-4f24-ae7f-3953967ea72b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.429090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:18.471052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:18.472157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:18.473475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:18.474065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:18.476253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:18.489026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05d59dc7-49c6-465c-895a-c86227fccffa]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137430, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.529290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9dbf3cc-5d6d-4ebe-8ab2-1ea8aa8d8c2d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.560645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4df8c4d-ddb5-444b-9b36-0d769dbf8027]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.578527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[951c622b-078d-4497-a8d6-6aa76441cccf]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:18.578527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:18.578527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:18.578527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:18.578527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:18.578527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:18.578527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:18.578527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:18.578527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:18.578527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:18.578527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:18.578527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:18.578527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:18.580424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[951c622b-078d-4497-a8d6-6aa76441cccf]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:18.581520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:18.584052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:18.588095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62233ad4-6cc3-48e4-8aec-7c427e5da7e5]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.588876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:18.589196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:18.589446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:18.589779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:18.592056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d505c44-8ff2-48dd-aa30-77bab1828fe5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.593299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9cf2ff0-9b97-4303-b568-1be5b5cdef76]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.594456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9f37fa3-6f2a-443b-be62-e04a8c65782e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.601815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[093027bd-5995-41fa-bb09-8aa81354e4d0]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.622918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1220b785-33a6-4cb8-bd71-226b1bb45c16]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 277445, 'reachable_time': 42004, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137446, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.630301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:18.630622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c903dadd-b9a3-4a3a-9118-97cb3fcf6f80]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:18.632218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:10Z", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:18.636393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:18.650788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id c861a568-988e-40ae-a7bb-0a3190f37a5d. Wait time elapsed: 9.740 Jul 01 09:31:18.650788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id c861a568-988e-40ae-a7bb-0a3190f37a5d. Wait time elapsed: 9.741 Jul 01 09:31:18.650788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:18.650788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09711128-a12d-4ec9-8ad3-540f2956f235]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.684822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d424fed6-8d7c-4ec3-a7e5-d055b14ec5aa]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.685751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:18.687091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:18.687695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:18.688203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:18.714817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:18.715842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[447349c1-cdbe-4935-be93-068c5d6cb587]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.718459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1700cce5-53aa-4e33-a2db-272b6da7911b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.719723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:18.759310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:18.760381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:18.761428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:18.762710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:18.764206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:18.823275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5873f6c6-a4ec-473c-87f2-0f2cd0dd56e6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.877446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[10331e86-114d-4700-b256-334a76d80653]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:18.877446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:18.877446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:18.877446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:18.877446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:18.877446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:18.877446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:18.877446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:18.877446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:18.877446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:18.877446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:18.877446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:18.877446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:18.878801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10331e86-114d-4700-b256-334a76d80653]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:18.880903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:18.884446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:18.889007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a876dc22-280f-44b6-b291-53d597b100a8]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.890927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:18.891431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:18.892177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:18.892621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:18.893938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67785a97-6cbe-4606-af36-5f2d46b9549b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.894969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ff59c87-c585-4340-9774-9969f442e9fa]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.896909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f09c4b9-e6ae-4136-afc9-4d40545a8851]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.924501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ee98215-f333-40c4-bb26-22712924844c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 277474, 'reachable_time': 36768, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137464, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.935287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:18.935750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d660a776-0193-4cee-a208-21adf168a589]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:18.936829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1c2933de-fbc4-4252-b66f-8f745f6b03a3", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.74", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4adaa360-ab59-46f0-a744-2c71b0db08dd", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1350, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:94:b6:4c", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "601e18e3-d27b-4d21-a5c3-a4d7827fe67d", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:56Z", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:56Z", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:18.939122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:16Z", Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id 4e8d824c-7889-4025-984c-9cda79653220. Wait time elapsed: 21.157 Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id 4e8d824c-7889-4025-984c-9cda79653220. Wait time elapsed: 21.157 Jul 01 09:31:18.953876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:20.446487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.508s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:20.447086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:20.447714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:20.448037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:20.448443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:20.449883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[390408dc-00dc-4d26-839f-c710384b2336]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.473051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0286b12-7b4f-4267-81ba-dbdf0e4e5e59]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.519445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae93fa24-aa81-44a8-982e-fc825c8131d8]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.552062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b840c936-2c89-4c08-9927-876b4014ba4a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137495, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.575226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c7dabdb-9366-4734-b9fe-8ea92dd69a0d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.611320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c3aa0e9-e113-462c-b2ec-7f7d87f813ca]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.633038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-6d912e85-b514-4313-a9f4-298c5fc9b19e tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['00025501-514b-4b36-87f4-cf1ff29b34af'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:20.635180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id 7969ad22-b9e1-4f83-843e-925b221c208e. Wait time elapsed: 0.002 Jul 01 09:31:20.637219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id 7969ad22-b9e1-4f83-843e-925b221c208e. Wait time elapsed: 0.004 Jul 01 09:31:20.645066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:20.663235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06a0a97d-2658-4d46-953f-fbce860d98da]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.691589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5e5418e-ed83-4266-bf43-e82897818a13]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.692381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:20.693054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:20.693506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:20.693938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:20.716397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:20.717036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba163202-7124-428f-b532-f037db370f7a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.721501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[284d2fb4-d748-4a21-9a89-6e8f7a070afa]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.722834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:20.764575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:20.765558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:20.766180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:20.766701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:20.767884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:20.844333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3757d569-fba5-4462-a4cc-c324290042b4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.884650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[59644949-5c16-4e60-aa39-be9a42e8eeb5]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:20.884650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:20.884650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:20.884650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:20.884650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:20.884650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:20.884650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:20.884650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:20.884650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:20.884650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:20.884650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:20.884650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:20.884650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:20.885930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59644949-5c16-4e60-aa39-be9a42e8eeb5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:20.887001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:20.889392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:20.893368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44cf3dd1-5ce7-433f-b6c9-d3767627cfb0]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.894270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:20.894723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:20.895206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:20.895699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:20.896969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3d5e0f9-f8d0-4033-a6fe-dd13e5a07f6a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.898056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf6d8b9d-05d9-411e-acf1-0b705b2d7310]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.899087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fb0d5ec-3b5e-4d53-a45f-8475b339ef8a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.926185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20f4f66d-5281-428d-b82f-66eb56e2e35b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 277681, 'reachable_time': 44676, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137513, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.935239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:20.935612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b298ab68-6d19-4f5b-b30f-9bc798c5ca43]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:31:20.936675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1c2933de-fbc4-4252-b66f-8f745f6b03a3", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.74", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4adaa360-ab59-46f0-a744-2c71b0db08dd", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1350, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:94:b6:4c", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "601e18e3-d27b-4d21-a5c3-a4d7827fe67d", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:56Z", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:56Z", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:20.939095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:19Z", Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:20.955990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3 Jul 01 09:31:21.363805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 6.254s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:21.364346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:21.366211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:21.367595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:21.368105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:21.369894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8a657c8-8e0f-401e-b2ab-297914479af7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.393073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8b87e4b-7cca-4083-8f19-6e57681a233e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.419190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6939efc-a512-4e11-b654-c5057af62b92]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.462601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[567b6fae-042b-4c35-904e-fd6d3637f852]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137516, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.509826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[632e029d-ac02-4003-a76f-eae5fa5e3936]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.542803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6af38812-28d0-40f9-a48d-91c1f9ec98b2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.577267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f03a8d86-abb0-4463-aa70-5e1183f2cd12]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.616052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[baede632-0525-459b-9ecd-6d268f1c3641]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.617170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:21.617452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:21.617724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:21.617997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:21.638611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:21.638868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0464093-d781-41bf-b93c-426a2e85d7ac]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.642874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e15a9388-a990-48ab-9422-4e22945b0035]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.644122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:21.688651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:21.689666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:21.690250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:21.690839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:21.691423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:21.768604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[509333b9-fb26-406a-9e3e-baaf1d56887c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.807847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4a884765-beb5-4814-b326-6cfbfa4937dc]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:21.807847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:21.807847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:21.807847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:21.807847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:21.807847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:21.807847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:21.807847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:21.807847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:21.807847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:21.807847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:21.807847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:21.807847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:21.809329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a884765-beb5-4814-b326-6cfbfa4937dc]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:21.809781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:21.812445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:21.816272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[239d5bc1-87b2-404e-96a9-9d70accbed9e]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.817406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:21.818232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:21.818628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:21.819173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:21.820391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b1a4189-324d-4337-a07d-a36ba222f9d6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.821493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[718f6aab-f920-4b85-bcfb-cbb9985ca95a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.822590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[638fe11b-5d2d-483c-aef9-d48d905705fa]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.858640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fff0cb9-1956-4f0e-96cd-eeb754778c9a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 277772, 'reachable_time': 15328, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137534, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.861137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:21.861585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36e69c4d-03f1-4291-9936-0d95c113f3e5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:21.863092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2c5c1a36-bdc1-4847-9f8f-e598677fd30a", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.206", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "5cd44a2d-bdd8-4364-8080-ef931dc139c1", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1347, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dd:94:6c", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0e6c39b8-7e9c-439c-86ab-2385f592e29b", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:53Z", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:53Z", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:21.868432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:17Z", Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:21.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:21.883948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 4e9c69aa-93d9-43ae-87c3-da0f4ae9620e. Wait time elapsed: 24.335 Jul 01 09:31:21.883948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 4e9c69aa-93d9-43ae-87c3-da0f4ae9620e. Wait time elapsed: 24.339 Jul 01 09:31:21.883948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:22.327848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.690s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:22.328116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:22.329084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:22.329405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:22.329682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:22.330992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[734fd43e-2ec1-48d3-aa3e-aed482dffc6c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.361961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a754a226-8662-49e1-857b-30abdb62b501]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.398795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60e1a9ed-4c2f-4412-b009-3c0b9b06256c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.461567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14a28749-ddc4-40d7-8880-e0660a511ea7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137537, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.495049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46b0af65-04a0-4e9c-9484-7c8ddf5644da]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.543482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2f31cb0-200c-4fa9-8dc5-9d0d06d9dbcf]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.595078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8547c5cf-deac-4613-b0ed-ff90a50a417e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.658141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eebb7e20-8b99-49b2-9f57-9843f4072992]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.658755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:22.659026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:22.659289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:22.659737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:22.697175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:22.697603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c6ffc35-7232-4448-877b-25257f1626d0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.701606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94659395-70ea-4357-859c-65c7a1f5129f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.702882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:22.742221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:22.742901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:22.743319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:22.743964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:22.744359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:22.778770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5cc8378-153f-4625-bca3-35ce78ad207a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.823058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[458438ee-d219-4102-9bba-99fce00075bf]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:22.823058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:22.823058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:22.823058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:22.823058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:22.823058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:22.823058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:22.823058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:22.823058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:22.823058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:22.823058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:22.823058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:22.823058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:22.824113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[458438ee-d219-4102-9bba-99fce00075bf]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:22.825701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:22.828260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:22.830039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ccb897d-af20-457f-b20a-9a887be54e78]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.830673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:22.831009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:22.831260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:22.832132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:22.834785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f337fa87-fc34-4e64-9fd4-6841b258976c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.836272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf7a3849-7721-45e4-b1bb-7f2904718c20]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.837345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3af2236d-8bd1-4ede-a22f-a2e718d25080]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.867399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01a9c9f9-dcb1-4053-9aa9-6abadeccc7f5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 277872, 'reachable_time': 16220, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137555, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.882382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:22.882619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c10ac52-6818-4336-a995-82ad1200ee68]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:22.884070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:10Z", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:22.888157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:22.908244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id 79a52ab3-eb65-4816-b403-5c714d0468f7. Wait time elapsed: 18.210 Jul 01 09:31:22.909384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id 79a52ab3-eb65-4816-b403-5c714d0468f7. Wait time elapsed: 18.210 Jul 01 09:31:22.910140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:23.007066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.124s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:23.007567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:23.009695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:23.010028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:23.010311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:23.011950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[104ed590-2a96-4928-a267-0c8a80138da7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.029564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f44b6737-91a1-4718-98db-3f5673837d4f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.055330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0f28c1a-5b81-41d4-8d75-89a19552d222]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.112145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db502731-1180-4e45-ac54-332ba4107b27]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137558, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.142768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe4f2b60-da3f-46f9-a557-b2a03caa0c4a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.187471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b64d206-4249-4844-bcc5-a5172bad455e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.219442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fb86d44-ba60-4905-9e10-16c5f5ddea07]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.252812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8535ff5-0f21-4676-8bdf-28b5795c20ae]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.253934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:23.254220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:23.254485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:23.255124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:23.281978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:23.285364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e685170c-aed5-474c-8675-aa128abe1898]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.288534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7456970d-0445-480f-ace0-f144e81bc82b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.290866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:23.328396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:23.329473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:23.330409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:23.332315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:23.333097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:23.392177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1663476-2501-4746-87a7-bd19373aa4e9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.452194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4dd0edfa-1569-40a6-9521-0e4031cb41eb]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:23.452194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:23.452194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:23.452194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:23.452194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:23.452194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:23.452194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:23.452194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:23.452194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:23.452194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:23.452194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:23.452194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:23.452194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:23.453300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4dd0edfa-1569-40a6-9521-0e4031cb41eb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:23.454539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:23.459397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:23.461251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48db558d-f2d6-4618-893a-64c69b344b06]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.461407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:23.461865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:23.462354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:23.462581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:23.464142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e625899-5582-4376-8f2e-c56c12ba1708]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.465659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[171159ec-3960-4694-a076-d088a1483c8b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.466539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16a08a4e-78fd-4041-830f-06a4c7379616]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.504050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb9e14d2-6cfb-43a0-89e1-5241466de209]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 277937, 'reachable_time': 28220, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137576, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.514638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:23.514921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb19eab9-7ecb-41c7-8471-11aa29ad1cd6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:23.515965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2c5c1a36-bdc1-4847-9f8f-e598677fd30a", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.206", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "5cd44a2d-bdd8-4364-8080-ef931dc139c1", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1347, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dd:94:6c", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0e6c39b8-7e9c-439c-86ab-2385f592e29b", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:53Z", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:53Z", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:23.519413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:21Z", Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 7b94701f-1773-4678-a963-a5ac13db9ced. Wait time elapsed: 63.503 Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 7b94701f-1773-4678-a963-a5ac13db9ced. Wait time elapsed: 63.503 Jul 01 09:31:23.534934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:24.726903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.199s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:24.727558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:24.729097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:24.731107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:24.731667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:24.738282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b81323d4-4038-431d-97b9-53d10b2d6f5f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:24.759599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0f3cb7d-03f1-412b-bb74-89c28946953f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:24.795585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2dcb05e8-c1fc-4a2e-94fd-d6494d6019d1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:24.842809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60a30caf-1cc1-48cc-a5a4-9cb81386a0ee]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137581, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:24.873320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f515a989-fe25-4b0b-a3b0-da7fec294365]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:24.900416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57f162fc-7112-4b5c-aa47-7dc9a8290935]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:24.955308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[321bb709-c9ff-40e8-bba7-a4ba17653581]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:24.984534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90db6942-7941-4550-823b-a643b6cb55b2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:24.986471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:24.986934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:24.987426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:24.987887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:25.015937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:25.016860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bda4b7b-e389-4c81-ae9f-30223cb093e0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:25.019948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6185be76-89c8-42ba-8d5e-ccac9812daec]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:25.021268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:25.048872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:25.049746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:25.050289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:25.050829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:25.051389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:25.126336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a82360fe-f647-4748-a66a-6d48fcba3481]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:25.172345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2d2739f7-3baa-4b80-82e2-fc273dd09574]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:25.172345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:25.172345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:25.172345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:25.172345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:25.172345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:25.172345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:25.172345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:25.172345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:25.172345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:25.172345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:25.172345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:25.172345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:25.173860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d2739f7-3baa-4b80-82e2-fc273dd09574]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:25.174763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:25.176228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:25.177980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2dc15236-0f23-41bb-b611-8b33a45fc259]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:25.177980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:25.177980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:25.178215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:25.178702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:25.180086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ebc2f332-7ff7-4d92-a766-c70161cc44cc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:25.181448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff0e55a3-4232-4d9d-8101-47f1e1ce8df8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:25.182394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[522c2f7f-c04b-4437-8ef2-7fb3d4553c39]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:25.209024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b8e3675-ddb7-45da-8c6a-cf68f65d8754]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 278110, 'reachable_time': 38412, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137602, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1be5d5bf-c227-48eb-8331-c4d74f182129]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:25.216155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2c5c1a36-bdc1-4847-9f8f-e598677fd30a", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.206", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "5cd44a2d-bdd8-4364-8080-ef931dc139c1", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1347, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dd:94:6c", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0e6c39b8-7e9c-439c-86ab-2385f592e29b", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:53Z", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:53Z", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:21Z", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:25.222352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3 Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 4e9c69aa-93d9-43ae-87c3-da0f4ae9620e. Wait time elapsed: 27.669 Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 4e9c69aa-93d9-43ae-87c3-da0f4ae9620e. Wait time elapsed: 27.670 Jul 01 09:31:25.247027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:25.394654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-adb6da46-cab0-46a7-9041-d99aa27576f6 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['00025501-514b-4b36-87f4-cf1ff29b34af'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:26.961804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.052s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:26.962879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:26.964883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:26.969238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:26.971265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:26.983824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[385b0f20-465b-467e-9a7a-833558eaa646]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.010502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3f611d0-5510-436d-b39d-632d7788f389]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.048316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b6b80a8-da5f-40ce-a414-281f24d1720e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.099944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4735ae2a-487b-4007-bf7e-b118b531f09f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137605, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.127431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[304aa908-748e-4c62-9402-90cc5bfc2a46]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.157272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e90212f8-68f1-43c2-9664-522d247ffc3a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.191648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5633beaf-b043-424d-9609-35d846a5af2d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.226441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fff64c98-2358-4de0-ac27-5dc9f01b043c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.227297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:27.228736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:27.229163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:27.229610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:27.255924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:27.256425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[511090d6-dc34-4658-bb96-092bb680c4fe]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.259187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca9b82f6-7975-4a1f-bb82-3b71adeb33ba]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.260405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:27.294575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:27.295432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:27.296088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:27.296623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:27.297152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:27.346847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cf0733a-6de3-4fe7-b6f9-324d25728e30]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.386323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[58621f22-185b-4a9e-9390-4712f3bbef34]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:27.386323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:27.386323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:27.386323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:27.386323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:27.386323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:27.386323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:27.386323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:27.386323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:27.386323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:27.386323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:27.386323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:27.386323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:27.387631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58621f22-185b-4a9e-9390-4712f3bbef34]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:27.387916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:27.391073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:27.393193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18e7969d-6e13-42f6-b440-8425bb423d82]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.393920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:27.394241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:27.394508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:27.394828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:27.396053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbfddaaa-78fe-47d7-8cd4-2819f7916606]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.397105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c62904f-aec7-45ca-90fb-ddfff4e5c427]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.398099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9941fbb-964f-4eb7-aeff-323ad510acff]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.424108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd0cc21d-688c-4d41-bcc1-f7a4256c6c29]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 278335, 'reachable_time': 16088, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137624, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.428670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:27.428670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[943a9bc5-b13f-4eed-8b89-a0cd578741ac]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:27.437220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:10Z", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:27.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id c861a568-988e-40ae-a7bb-0a3190f37a5d. Wait time elapsed: 18.542 Jul 01 09:31:27.459863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id c861a568-988e-40ae-a7bb-0a3190f37a5d. Wait time elapsed: 18.542 Jul 01 09:31:27.459863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:28.343529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.904s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:28.344117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:28.344736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:28.345054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:28.345336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:28.346828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff374a6e-fd59-46c7-96fd-c2937342c774]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.370478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ab7dd99-7de1-4b75-9166-ee654455b17c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.395874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56d71a13-401f-4930-85ae-4040496ec3ec]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.453905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[045e31b4-387e-4cf9-a0f7-4be0e221d7d0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137627, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.476287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c3eee71-703c-4027-a244-fb4ba6360035]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.501588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[885da4cf-3a17-49d7-ab00-2b41c8513daf]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.510451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 7.865s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:28.510799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 00025501-514b-4b36-87f4-cf1ff29b34af info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:28.513121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (00025501-514b-4b36-87f4-cf1ff29b34af), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:28.513121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:28.513427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:28.516746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51e703d7-8833-4062-a890-8af2420436ae]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.536388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[411d726c-cf76-4869-bb8c-5961eebd15c8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.555670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11520c5d-81b5-4fbd-bb7a-03033fe8653e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.580149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1b8b25d-ae46-4570-91b8-c2e02bf3ee6c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.581025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:28.581500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:28.582193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:28.583175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:28.614686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:28.615534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46839bb5-3624-40a9-8226-8c30159b6ed0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.619391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73e676db-906e-4630-8356-c6ed378a2a41]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.619956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9d52508-bad8-49fb-98fd-0cf65c6b139f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.621336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:28.649542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:28.649730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:28.650639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:28.651102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:28.651639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:28.712596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0949dd13-7919-4448-91ee-dae10fee3de0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.766566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97f10802-f409-404e-9e76-07c12d59cdde]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137638, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.787676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4f7fd4db-d541-478a-aec1-2f3329ca3cab]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:28.787676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:28.787676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:28.787676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:28.787676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:28.787676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:28.787676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:28.787676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:28.787676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:28.787676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:28.787676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:28.787676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:28.787676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:28.790601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f7fd4db-d541-478a-aec1-2f3329ca3cab]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:28.791619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:28.794206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:28.799315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb24a9a4-3ee8-4c12-adb7-4f116bc69dfb]: (4, ['qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.800071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:28.800373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:28.800624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:28.800955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:28.803761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4508a747-2698-44c7-a46b-45a163257a9d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.804251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9a5af98-3d7d-4386-89e1-0e299e00780b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.811621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef6f4d2d-cc6b-4037-bd87-8f8685e01082]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.819765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aedf7178-4fbf-4c0c-882d-aa5157456279]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.852579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4eecddb-cec3-44f7-846c-b3c4527e3963]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.857218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20cffdd5-4f1c-4b7a-b023-f2a565ab34ab]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 278471, 'reachable_time': 16708, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137652, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.868040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:28.868653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c64815c-9214-4985-a4f4-d9185c42d2dd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:28.869912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:10Z", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:28.874427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:28.883838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id 79a52ab3-eb65-4816-b403-5c714d0468f7. Wait time elapsed: 24.187 Jul 01 09:31:28.883838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id 79a52ab3-eb65-4816-b403-5c714d0468f7. Wait time elapsed: 24.188 Jul 01 09:31:28.883838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:28.888708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74ea28ea-6b78-408f-9923-f78e1df9b2a6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.922419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77ad0d71-8f00-4d95-a4d0-d9b049283047]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.925889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:28.926374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:28.926841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:28.927315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(5797795b-9c38-4a89-aaaf-36f3d27d7977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:28.935206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-981d5f78-c225-465d-8281-ff72752d02b0 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['00025501-514b-4b36-87f4-cf1ff29b34af'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:28.959978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:28.960583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffd22f96-082c-48e0-a4d0-2d8e4605e102]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.965749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2637fb8-e40a-4036-aa8f-bc645f9d67f1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:28.967520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-5797795b-9c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:28.968045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:31:28.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-5797795b-9c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:28.970167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:28.970868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:28.971581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:28.972769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-5797795b-9c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '5797795b-9c38-4a89-aaaf-36f3d27d7977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6d:28:93'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:29.031310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a7fe095-be1f-4480-aa3c-1a55066a1b4f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:29.064403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9481be4c-f90b-4974-8c6c-ea46d3ce6bff]: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:29.064403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:29.064403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:29.064403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:29.064403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:29.064403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:29.064403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:29.064403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:29.064403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:29.064403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:29.064403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:29.064403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:29.064403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:29.065273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9481be4c-f90b-4974-8c6c-ea46d3ce6bff]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:29.065683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:29.067264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:29.071041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7342be27-2b98-47cb-85f2-725705f2a552]: (4, ['qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:29.071041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:29.071041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:29.071041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:29.071041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:29.071396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0f23f5e-6401-49cd-847c-a6195c763885]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:29.072063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c28c191c-1e8e-4913-aab8-8a63308d8dc3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:29.073046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61b6d5a6-0730-4e0a-a856-64f3b25710af]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:29.103240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98ab5a1c-1de7-4b22-80c0-db98867ee001]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 278501, 'reachable_time': 23400, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137666, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:29.106267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:29.106758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[536044ca-b5c1-44ff-8da6-aebe0a744c9a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:29.109845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersTest-router-1948506809", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:15Z", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:23Z", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5797795b-9c38-4a89-aaaf-36f3d27d7977", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "e68c188b-7855-4484-9148-26099a7fbe2e", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6d:28:93", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1393, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:16Z", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:24Z", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88ae0bcc-6d24-4f81-9919-60894e295014", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1321ae28-b9b5-4d4e-9033-c61d7eafaa7b", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:ee:ee", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1404, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "1d2b103e-e328-4cc4-960c-bf1a25d2d121", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.21.0.1", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:22Z", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:24Z", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 2, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1d2b103e-e328-4cc4-960c-bf1a25d2d121", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.21.0.0/28", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.21.0.1", Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:29.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:29.124613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id 57c9e211-292d-47f1-8ae0-2b9cf1e7653e. Wait time elapsed: 3.729 Jul 01 09:31:29.125041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id 57c9e211-292d-47f1-8ae0-2b9cf1e7653e. Wait time elapsed: 3.729 Jul 01 09:31:29.125346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:32.006186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.881s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:32.007278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 00025501-514b-4b36-87f4-cf1ff29b34af info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:32.008533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (00025501-514b-4b36-87f4-cf1ff29b34af), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:32.009064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:32.009548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:32.012344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee78a589-96bf-435a-85e5-90c052bea21a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.019574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 6.805s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:32.020113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:32.022068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:32.023037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:32.023548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:32.024960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1de04d42-21fa-4288-b08e-95fcc949f303]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.034748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2049fae-f604-4594-abf5-711576f32bca]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.046497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[978fe1b8-7039-4850-b6cd-2b3e60f9906b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.070751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[335ab32b-f20d-4583-b29e-eb2e86315d29]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.081517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf974912-663e-427c-8690-41ca5e7da540]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.133285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[495d4329-293a-448f-91c6-bc38e4a65310]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137737, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.137146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[212d1392-b4ff-4d1a-bf96-47b8f2a13949]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137738, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.176880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a84c3f63-91cd-4dc4-8fc4-798190576baf]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.177665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4460385-7bce-4b1d-9b9e-09a81eaaf06b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.217197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9524ae2c-09a2-4e83-b303-18fb635d4c0b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.217532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3841d20-318c-4cad-a815-151aeb8dd43c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.249240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d40897ac-30ff-43d9-9970-40f56a836198]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.257460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d40656f-2a0e-4d8d-8901-54ae42ab1f7a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.289178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e709a6a-b53b-410f-8e5d-6e8fe811d1dd]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.289743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:32.290023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:32.290599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:32.290886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:32.302177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[880cbb65-7648-499e-b6b9-625ba19a313f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.302749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:32.303577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:32.304254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:32.304892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(5797795b-9c38-4a89-aaaf-36f3d27d7977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:32.324143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:32.324864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fef875d2-3c98-4703-b2dc-b436fe8975f1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.331097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45e97c9b-2e0f-4fb2-aad7-af129c3a0f6f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.332181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:32.346054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:32.346355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9550d454-e653-40af-ad2c-6eed85928bda]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.348687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c37b629f-d810-404f-b7e3-e82aa76d5c9a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.368859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-5797795b-9c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:32.422858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:32.423972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:32.424603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:32.425279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:32.426948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:32.472964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-5797795b-9c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:32.484684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:32.488783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:32.489186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:32.489585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-5797795b-9c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '5797795b-9c38-4a89-aaaf-36f3d27d7977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6d:28:93'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:32.503314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88a84efc-fe5a-40c4-9ba0-b690af22e829]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.547109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a511dbb-32f9-4d24-9b85-a4dbf4012c9b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.593257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a15a461f-d6cc-40da-9482-2424f7cc06cf]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:32.593257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:32.593257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:32.593257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:32.593257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:32.593257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:32.593257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:32.593257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:32.593257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:32.593257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:32.593257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:32.593257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:32.593257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:32.594243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a15a461f-d6cc-40da-9482-2424f7cc06cf]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:32.594396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:32.596108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:32.599222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[926ec111-86f5-4740-aa10-f235becbe156]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.599987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:32.600311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:32.600566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:32.600909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:32.606587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[913f672e-4305-4b0b-b667-6828ea52629a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.609087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b5c277c-c28d-45d0-8cdc-54709dfe51ce]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.610291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[84ea63e7-ac09-423b-ad1f-0593db2e59d6]: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:32.610291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:32.610291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:32.610291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:32.610291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:32.610291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:32.610291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:32.610291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:32.610291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:32.610291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:32.610291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:32.610291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:32.610291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:32.611551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84ea63e7-ac09-423b-ad1f-0593db2e59d6]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.612050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52c0f360-4581-43af-9b7a-32d80e9f7fc4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:32.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:32.615423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:32.622727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00f424ed-b6c6-499f-a838-52b69a603c49]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.623717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:32.624138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:32.624453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:32.625063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:32.628025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[244c49df-ad4e-45e5-8c5a-26b09583d7da]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.629190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7371e59-9271-47fc-a469-bec93e5190df]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.630313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7e736dc-d41d-498d-a25e-7f659295b19d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.656071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc7b1c99-54db-40f3-8451-0636be774492]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 278839, 'reachable_time': 38972, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137774, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.658920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:32.659525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08f68f4e-8bd4-40dc-a1b2-7c416e6c3252]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:32.661752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2c5c1a36-bdc1-4847-9f8f-e598677fd30a", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.206", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "5cd44a2d-bdd8-4364-8080-ef931dc139c1", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1347, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dd:94:6c", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0e6c39b8-7e9c-439c-86ab-2385f592e29b", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:53Z", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:53Z", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:32.665310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:21Z", Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29b8734b-a0dc-4aae-bdba-5a383796ae15]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 278839, 'reachable_time': 30024, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137775, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 4e9c69aa-93d9-43ae-87c3-da0f4ae9620e. Wait time elapsed: 35.133 Jul 01 09:31:32.685271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 4e9c69aa-93d9-43ae-87c3-da0f4ae9620e. Wait time elapsed: 35.133 Jul 01 09:31:32.687827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:32.691111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:32.691609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f251d03-d82f-420a-9761-1aa04b520859]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:32.693860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersTest-router-1948506809", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:15Z", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:27Z", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5797795b-9c38-4a89-aaaf-36f3d27d7977", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "e68c188b-7855-4484-9148-26099a7fbe2e", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6d:28:93", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1393, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:16Z", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:24Z", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88ae0bcc-6d24-4f81-9919-60894e295014", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1321ae28-b9b5-4d4e-9033-c61d7eafaa7b", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:ee:ee", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1404, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "1d2b103e-e328-4cc4-960c-bf1a25d2d121", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.21.0.1", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:22Z", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:30Z", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1d2b103e-e328-4cc4-960c-bf1a25d2d121", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.21.0.0/28", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.21.0.1", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd476201-48c3-4b34-aa8a-442ae669146d", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "857cbe40-ba64-4a8a-a0b0-f7b508dcac1e", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:5e:09:83", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1417, Jul 01 09:31:32.698361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "12a6236e-78c9-4c3d-9ac9-6903ba45fccf", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.22.0.1", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:27Z", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:31Z", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "12a6236e-78c9-4c3d-9ac9-6903ba45fccf", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.22.0.0/28", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.22.0.1", Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id d4a51edc-cf01-4416-9531-eaeece911e96. Wait time elapsed: 3.774 Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id d4a51edc-cf01-4416-9531-eaeece911e96. Wait time elapsed: 3.774 Jul 01 09:31:32.714175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:33.698550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ce973c03-da36-48ce-b075-fa33db8f3258 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['00025501-514b-4b36-87f4-cf1ff29b34af'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:34.400403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.519s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:34.400568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:34.401594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:34.401804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:34.402375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:34.403467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ded270c-faab-4009-8ce3-009582699ff7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.436235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eadf5d34-fbfe-415b-b6e1-c156da6945fe]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.481547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b23a29c-1450-479e-bc74-65a3f1291e74]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.548656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[942e7eac-e7db-45a6-a722-29981f0747a4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137779, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.584404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2011a707-8027-4e16-8fda-0df83083a9ff]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.622533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[915c2e67-f46d-4d0a-84c5-99c1b27b56b4]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.659490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[485ecf5a-0bd1-4e7c-93ef-e2c6641d339c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.715893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7afa47dc-9e06-4677-b384-2ad03acbd12a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.717734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:34.718189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:34.718660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:34.719122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:34.730986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.018s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:34.731655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 00025501-514b-4b36-87f4-cf1ff29b34af info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:34.732727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (00025501-514b-4b36-87f4-cf1ff29b34af), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:34.733403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:34.733697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:34.736683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d537e4a-44b0-4e39-8cdb-154db265f74f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.753348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:34.753899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[042c834a-983c-41f0-a903-c4de2100ba7a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.759877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9aac3a95-7ef5-47da-94e1-6552de8767ea]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.762177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:34.774417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a769917-7234-469f-9df4-d324987b88fb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.796759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:34.797423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:34.798046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:34.799222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:34.799837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:34.810460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85093504-159e-427b-b6b6-dd7d7fff50dd]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.864976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[021ee55f-f5ae-4390-8193-247446a762f9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 3627305624, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.882803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8fb74154-cde8-4e86-af05-238f298a7054]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137792, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.904997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[708de49f-8c0e-4a23-9d1a-feb582cf9903]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:34.904997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:34.904997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:34.904997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:34.904997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:34.904997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:34.904997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:34.904997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:34.904997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:34.904997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:34.904997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:34.904997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:34.904997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:34.906450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[708de49f-8c0e-4a23-9d1a-feb582cf9903]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:34.907611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:34.910720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:34.914584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f17a946-a015-4bf7-860c-7b2c8ebf1d56]: (4, ['qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.915340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:34.915715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:34.915973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:34.916318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:34.917383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee37e061-8ff6-4de3-b327-4e307c130944]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.919022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eae5bdb3-2576-4baa-8ac6-5a18f90f0234]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.924534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37873954-a422-4fae-8e23-9bb63d1129f7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.925841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcae9924-ab52-4219-96d7-59b0a547aabc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.956142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d38990ca-e6d0-4f20-9853-841997831056]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.967678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce94ab81-6fed-478c-aa7c-ce96e4d9ee3f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 279081, 'reachable_time': 26604, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137806, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.973289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:34.973377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82f4f02c-dc48-43e4-bee1-46838622462c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:34.974426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:10Z", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:34.977179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id c861a568-988e-40ae-a7bb-0a3190f37a5d. Wait time elapsed: 26.079 Jul 01 09:31:35.001898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id c861a568-988e-40ae-a7bb-0a3190f37a5d. Wait time elapsed: 26.079 Jul 01 09:31:35.001898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:35.018683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d54104ec-bdff-43b1-85ca-58e8999ccc7e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:35.051748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a67954cd-a1ac-45b2-a869-0bc28ae31cd9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:35.052695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:35.053179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:35.053660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:35.054197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(5797795b-9c38-4a89-aaaf-36f3d27d7977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:35.078838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:35.079970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37199ef8-d7b0-4f29-96e4-87fb7a03f1d8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:35.085105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1f007c5-b62a-405e-8a81-16c454d4c221]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:35.086475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-5797795b-9c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:35.106135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:31:35.112897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-5797795b-9c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:35.117406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:35.117821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:35.118212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:35.118654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-5797795b-9c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '5797795b-9c38-4a89-aaaf-36f3d27d7977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6d:28:93'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:35.145165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8fda8c6c-505e-4e80-bd7d-371c9f472e4c tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['00025501-514b-4b36-87f4-cf1ff29b34af'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:35.167606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ebbb9278-9b62-4ee7-b405-f159ced57cf1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:35.208502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[67a110b2-8d36-463f-98c1-74d8d3803b21]: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:35.208502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:35.208502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:35.208502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:35.208502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:35.208502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:35.208502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:35.208502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:35.208502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:35.208502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:35.208502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:35.208502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:35.208502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:35.210626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67a110b2-8d36-463f-98c1-74d8d3803b21]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:35.211614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:35.213136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:35.215763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7afbee84-bda7-4b55-9588-6f0865318ca6]: (4, ['qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:35.215763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:35.215763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:35.216099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:35.216272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:35.217401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09b0e7b7-5563-437f-aa4e-b4fe1fa24c86]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:35.218695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a113a31f-7802-4d01-9efc-83648cbf568e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:35.220579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4cd4ede-141e-4588-96dc-d4d6fbe382dd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:35.240441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e4e2480-39d3-444c-808d-ba2d92b0870f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 279114, 'reachable_time': 42936, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137821, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:35.243004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:35.243287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0241395c-04ae-4e0c-9ffb-2b2df156f8e4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:35.244401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersTest-router-1948506809", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:15Z", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:32Z", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5797795b-9c38-4a89-aaaf-36f3d27d7977", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "e68c188b-7855-4484-9148-26099a7fbe2e", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6d:28:93", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1393, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:16Z", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:24Z", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88ae0bcc-6d24-4f81-9919-60894e295014", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1321ae28-b9b5-4d4e-9033-c61d7eafaa7b", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:ee:ee", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1404, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "1d2b103e-e328-4cc4-960c-bf1a25d2d121", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.21.0.1", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:22Z", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:30Z", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1d2b103e-e328-4cc4-960c-bf1a25d2d121", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.21.0.0/28", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.21.0.1", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd476201-48c3-4b34-aa8a-442ae669146d", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "857cbe40-ba64-4a8a-a0b0-f7b508dcac1e", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:5e:09:83", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1417, Jul 01 09:31:35.247102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "12a6236e-78c9-4c3d-9ac9-6903ba45fccf", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.22.0.1", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:27Z", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:31Z", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "12a6236e-78c9-4c3d-9ac9-6903ba45fccf", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.22.0.0/28", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.22.0.1", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "eb86aef0-b92b-497d-b25f-3e6e5be71488", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "112e8154-6630-49ca-a18a-832c1f69d6f8", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:74:90:57", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1427, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "9afa296b-2293-483e-940c-4ff538e1e3e3", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.23.0.1", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:31Z", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:34Z", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9afa296b-2293-483e-940c-4ff538e1e3e3", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.23.0.0/28", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.23.0.1", Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id f61b1eb1-08d1-4462-a60c-25c6a8b4fc8c. Wait time elapsed: 1.547 Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id f61b1eb1-08d1-4462-a60c-25c6a8b4fc8c. Wait time elapsed: 1.547 Jul 01 09:31:35.263189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:36.210724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.964s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:36.211679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 00025501-514b-4b36-87f4-cf1ff29b34af info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:36.212788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (00025501-514b-4b36-87f4-cf1ff29b34af), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:36.213316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:36.213778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:36.215993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[078978d9-3691-46dd-8a63-9c20e1765c29]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.254843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87317ca6-d29b-4dfb-8bb8-a10413e7c6d6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.285035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70621674-da95-4004-bc97-dc152023a9c0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.384277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58b3ae0b-098e-48f7-aff7-62a36f2d17bc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137828, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.408225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac5d7c6c-1b4d-4ff2-a980-e6e9b7ec58c4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.429934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dace9ac7-963e-4b2b-8d1f-4d360d2ae667]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.466486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f71b138e-e705-4457-a358-eaeed20b7e12]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.507052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be174c6a-983d-49e5-8e51-c9dd11de2da2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.507862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:36.508374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:36.508838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:36.509766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(5797795b-9c38-4a89-aaaf-36f3d27d7977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:36.528829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:36.529430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84ca77f3-6ef3-4348-865c-3adbe1f38240]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.532318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e49da850-1f34-4b1d-b963-40980e4c1f4a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.533521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-5797795b-9c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:36.562477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-5797795b-9c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:36.563235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:36.563614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:36.564007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:36.564400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-5797795b-9c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '5797795b-9c38-4a89-aaaf-36f3d27d7977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6d:28:93'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:36.616263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acc5c6d2-e33d-41af-a2ea-82c2b385cfbf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3e94c799-3ab4-43e0-9231-7f220ea98709]: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:36.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:36.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:36.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:36.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:36.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:36.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:36.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:36.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:36.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:36.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:36.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:36.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:36.669129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e94c799-3ab4-43e0-9231-7f220ea98709]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:36.670468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:36.674153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:36.680470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-873597ba-0bab-45c6-b810-da2c97ab22c8 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['00025501-514b-4b36-87f4-cf1ff29b34af'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:36.681623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37504094-7297-42d9-8e24-97e51a045821]: (4, ['qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.684291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:36.684882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:36.685242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:36.685710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:36.686915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a8a2b2a-3825-4a9c-8248-06d80572d5fd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.688150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[889938b3-cdec-4e5f-843b-55b613b85495]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.689528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbfcc0ad-7a78-463c-bc61-5a4969c753bf]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.727543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51e10862-3762-47f8-88d1-faecceaa2da2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 279264, 'reachable_time': 42804, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137858, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.738355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:36.738946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[533f72a9-8f0c-4543-b1d6-9517dafc4f11]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:36.740465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersTest-router-1948506809", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [ Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "destination": "172.20.0.0/28", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nexthop": "172.20.0.2" Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "destination": "172.21.0.0/28", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nexthop": "172.21.0.2" Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "destination": "172.22.0.0/28", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nexthop": "172.22.0.2" Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "destination": "172.23.0.0/28", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nexthop": "172.23.0.2" Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:15Z", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:34Z", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5797795b-9c38-4a89-aaaf-36f3d27d7977", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "e68c188b-7855-4484-9148-26099a7fbe2e", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6d:28:93", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1393, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:16Z", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:24Z", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88ae0bcc-6d24-4f81-9919-60894e295014", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1321ae28-b9b5-4d4e-9033-c61d7eafaa7b", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:ee:ee", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1404, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "1d2b103e-e328-4cc4-960c-bf1a25d2d121", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.21.0.1", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:22Z", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:30Z", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1d2b103e-e328-4cc4-960c-bf1a25d2d121", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.21.0.0/28", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.21.0.1", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd476201-48c3-4b34-aa8a-442ae669146d", Jul 01 09:31:36.742959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "857cbe40-ba64-4a8a-a0b0-f7b508dcac1e", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:5e:09:83", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1417, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "12a6236e-78c9-4c3d-9ac9-6903ba45fccf", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.22.0.1", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:27Z", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:31Z", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "12a6236e-78c9-4c3d-9ac9-6903ba45fccf", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.22.0.0/28", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.22.0.1", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "eb86aef0-b92b-497d-b25f-3e6e5be71488", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "112e8154-6630-49ca-a18a-832c1f69d6f8", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:74:90:57", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1427, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "9afa296b-2293-483e-940c-4ff538e1e3e3", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.23.0.1", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:31Z", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:34Z", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9afa296b-2293-483e-940c-4ff538e1e3e3", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.23.0.0/28", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.23.0.1", Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id 6071126c-7304-473f-a329-654f7f4957e3. Wait time elapsed: 1.602 Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id 6071126c-7304-473f-a329-654f7f4957e3. Wait time elapsed: 1.603 Jul 01 09:31:36.760758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:38.632076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-1f7d29e8-56d8-4586-b64c-0b6fbf311de1 tempest-ServersTestManualDisk-1149109101 tempest-ServersTestManualDisk-1149109101-project] Got routers updated notification :['22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:38.953894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-be39ffab-43ce-4f83-b983-c6ef8b0cd4ba tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['00025501-514b-4b36-87f4-cf1ff29b34af'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:39.646344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.670s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:39.647347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:39.648942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:39.649433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:39.649865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:39.651676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4990907-3174-41e2-8822-171736e1a09c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:39.675599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ba76059-ffe6-43bb-af1b-dd946390daa0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:39.697635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66bb913a-9c8e-4e3e-b900-c7496684a44e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:39.736375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38a27483-0970-4af3-90eb-acdd289b832f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137882, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:39.759439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ade4601-d861-472c-acd7-ce172a12bb04]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:39.784335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[617d3f24-068a-4d6a-ae38-bc19e5dcc21f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:39.812670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ecb2493-adea-4c79-adb3-956ab5e6a955]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:39.848996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a06459a-4d21-4734-abfd-dd1680278a2c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:39.853581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:39.854011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:39.854435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:39.854872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:39.883948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:39.884883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96cf9451-8eda-4bf4-a11e-f14ab0e44c07]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:39.891535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[430931a2-6251-4edb-98df-f3b1d57949f6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:39.892796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:39.943701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:39.944374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:39.944769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:39.945154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:39.945536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:40.001589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfd2c1c1-4b63-4b1b-98cb-eb64f341a8be]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.061817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[52a9848c-a87c-41ce-be48-e6e7f7499711]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:40.061817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:40.061817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:40.061817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:40.061817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:40.061817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:40.061817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:40.061817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:40.061817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:40.061817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:40.061817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:40.061817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:40.061817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:40.063163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52a9848c-a87c-41ce-be48-e6e7f7499711]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:40.064238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:40.066516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:40.085468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4f4a4e9-188b-454e-8637-18a784dc7783]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.096005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:40.096421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:40.096568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:40.131908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:40.180592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 7.503s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:40.181313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:40.183491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:40.184078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:40.184697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:40.185390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[467f59b3-d7dd-4364-9759-26f0a2ac16aa]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.185915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bdf68af9-d002-4cd0-8692-0588fff0cd47]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.230451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57bcd691-f3c9-4a63-a058-96c59deee2c3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.230840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6a1391b-5d7b-4e70-8dd3-d5ae8bf06b92]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.254168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c72c95fb-9b67-4979-a09c-81617dd99e57]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.276187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[004750d3-b9a0-4a01-b5a1-447b7df5bd8c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.291829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9667fdf0-cb85-42db-8141-6436ec26c31d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 279600, 'reachable_time': 42544, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137931, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.310234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:40.310526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd99082c-a01b-4812-8c9f-af387a762571]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:40.312091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:40.315876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:40.329720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 1, update_id a7fdcc0b-44bf-4b2d-ab67-61ab20b6c71d. Wait time elapsed: 1.693 Jul 01 09:31:40.329720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 1, update_id a7fdcc0b-44bf-4b2d-ab67-61ab20b6c71d. Wait time elapsed: 1.693 Jul 01 09:31:40.329720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:40.340822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a5f3676-4364-4e07-a3bb-5f9e36a28cb3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137933, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.365612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51e9a484-3b67-4f0a-a98c-fbb041d3a06c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.426031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c62aa5fc-ac7b-4a4d-ae53-3ea484d7fbde]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.457693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16f3b436-0690-467c-a9c2-12e556259754]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.498991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30e18d96-9f4f-4923-8cb3-358ac7f70c82]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.499633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:40.499911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:40.500173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:40.500448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:40.530439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:40.530823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00cb0540-f2f3-42f2-b70d-15915a260b18]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.534022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da511aeb-90f7-4385-9ec1-85fd58b11406]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.535004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:40.566195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:40.567044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:40.567621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:40.568412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:40.568966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:40.625352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae537399-32b5-4424-b411-42f00eb08336]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.677539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3715783d-b15a-4691-ae11-46ed4fd680a5]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:40.677539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:40.677539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:40.677539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:40.677539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:40.677539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:40.677539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:40.677539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:40.677539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:40.677539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:40.677539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:40.677539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:40.677539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:40.679592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3715783d-b15a-4691-ae11-46ed4fd680a5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:40.680899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:40.684362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:40.688300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e935c7b0-75e0-4238-8acc-674a7e6497b2]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.689299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:40.689886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:40.690275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:40.690755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:40.691999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07c1f639-7ae8-4a6b-9379-5691de30cb79]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.693683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9e68d94-44ae-47b9-824f-d57465114106]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.697766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9fccfc1-bbe2-451b-8d27-c35efe9f6abc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.737166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e0eed4d-39e7-4c83-b30b-5ac54f8bd3ef]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 279660, 'reachable_time': 22348, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137951, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.749283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:40.753726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a72664c9-e177-4f7d-859e-739f601babbc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:40.755000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2c5c1a36-bdc1-4847-9f8f-e598677fd30a", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.206", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "5cd44a2d-bdd8-4364-8080-ef931dc139c1", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1347, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dd:94:6c", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0e6c39b8-7e9c-439c-86ab-2385f592e29b", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:53Z", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:53Z", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:40.759987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:35Z", Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:40.770571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:40.777727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 4e9c69aa-93d9-43ae-87c3-da0f4ae9620e. Wait time elapsed: 43.234 Jul 01 09:31:40.778122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id 4e9c69aa-93d9-43ae-87c3-da0f4ae9620e. Wait time elapsed: 43.234 Jul 01 09:31:40.778564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:41.453363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.127s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:41.453873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:41.455515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:41.455949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:41.456333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:41.457906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f8fb39a-c17d-4476-b801-faadc2c01803]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.473317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1a6caf0-9bc8-45f5-b41e-88315ddc60ef]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.495524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc66b84f-427e-4eac-b218-5997bdffd2d1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.532035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ae847a9-e378-443a-b66c-4ce54a46ceb1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137954, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.553479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3f1f1f7-1931-43d0-ab97-008d78362190]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.567191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fa52d56-381a-4c13-8dba-d97d386e66d7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.589858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0a31512-21a3-4e29-909c-0d4e89a55d9c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.612740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16cbd955-ac9c-4e50-b89f-32c78dfcc396]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.613675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:41.614223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:41.614553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:41.614894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:41.634429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:41.634751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6e2da21-cdf9-4a78-aae1-51c65c8995e2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.637810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07bdfd1c-1a0e-4102-b210-cb6cd9ed2d1c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.639989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:41.679033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:41.684202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:41.684743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:41.685139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:41.685532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:41.720726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[258ba30d-f5b5-4b36-964e-ba3d5a46a2d5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.763148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[84118e7e-fd88-49dd-b142-92416a447eb9]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:41.763148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:41.763148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:41.763148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:41.763148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:41.763148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:41.763148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:41.763148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:41.763148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:41.763148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:41.763148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:41.763148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:41.763148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:41.764130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84118e7e-fd88-49dd-b142-92416a447eb9]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:41.764519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:41.765989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:41.767875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[511ba329-1909-450d-b3d6-839c0d10e6f4]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.767875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:41.767875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:41.767875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:41.769309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:41.770520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ad4ab2f-07e0-4b18-934e-0e17b4cbc976]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.772961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2c7279e-edd3-4a93-8de6-0625e54810eb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.774063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbd1c656-f4c7-4c52-b35c-f652a3a60198]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.804457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[810fff06-bb57-4d3d-871d-72fd7524b539]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 279779, 'reachable_time': 40128, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137973, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.814385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:41.814736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d099a0a8-b280-4cdb-9f34-37f9770ed41d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:41.815907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "caef9192-df58-4fac-b455-ffd542999647", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.86", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "70a27689-ef53-4838-bc3d-7d03c0119dc4", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1384, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:68:a6:05", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9ffd697e-81f0-41cc-ab54-952c8a1bbb00", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:11Z", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:41.819919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id 79a52ab3-eb65-4816-b403-5c714d0468f7. Wait time elapsed: 37.140 Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id 79a52ab3-eb65-4816-b403-5c714d0468f7. Wait time elapsed: 37.141 Jul 01 09:31:41.838912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:42.210434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.467s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:42.210921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 00025501-514b-4b36-87f4-cf1ff29b34af info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:42.213657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (00025501-514b-4b36-87f4-cf1ff29b34af), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:42.214242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:42.215227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:42.217125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fb05d51-87e1-4266-a712-f2150c92aa89]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.243465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[465b66a5-44db-44f6-b67c-66c6ad1dc96a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.289950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d431eaad-fe7b-43fd-9f3f-ece5e496cb93]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.348375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b760710-6a79-437d-9b3b-96b87d8850d0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137977, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.377130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b29632f1-fbb8-480d-ae9a-4dcb34f7d81c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.409944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68c9a3e1-5eef-4bb5-a223-d130b3154ea9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.463276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e2b5603-a43a-4ec5-b60e-85c8abffd84a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.500308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0a6e093-9f0a-4745-9d7d-89a6bc3c472f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.501031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:42.501535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:42.501943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:42.506322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(5797795b-9c38-4a89-aaaf-36f3d27d7977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:42.549350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:42.549926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f06a39a3-abe2-4324-aff9-d7c7d6e2cc27]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.553336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9381bd1-9856-4369-9103-4b8b8feb1b4b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.554549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-5797795b-9c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:42.594289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-5797795b-9c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:42.595243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:42.597220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:42.597822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:42.599334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-5797795b-9c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '5797795b-9c38-4a89-aaaf-36f3d27d7977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6d:28:93'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:42.645867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a465764e-441e-49b3-88bc-bd1c63aeba0d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.699203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e07fdd76-e466-4c64-bde1-871eabb847db]: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:42.699203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:42.699203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:42.699203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:42.699203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:42.699203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:42.699203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:42.699203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:42.699203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:42.699203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:42.699203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:42.699203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:42.699203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:42.701274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e07fdd76-e466-4c64-bde1-871eabb847db]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:42.702385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:42.705614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:42.709349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b57da7c4-d31e-42c2-ab6b-58f5445fdde6]: (4, ['qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.710672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:42.711561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:42.711824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:42.712232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:42.713693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a076f41d-4ea5-499d-9cdf-3392c55501f7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.716187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a84dbc40-c744-40ab-8145-df6f9cd0b621]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.719916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8663895c-ec39-48cb-9b82-71312c3236c2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.758875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03f10b9f-2b67-4115-8b20-8e99f937518d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 279861, 'reachable_time': 38552, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 137995, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.769099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:42.769573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6798616a-a2d6-49ec-a2f3-2e8a961de76a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:42.770859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersTest-router-1948506809", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:15Z", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5797795b-9c38-4a89-aaaf-36f3d27d7977", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "e68c188b-7855-4484-9148-26099a7fbe2e", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6d:28:93", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1393, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:16Z", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:24Z", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88ae0bcc-6d24-4f81-9919-60894e295014", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1321ae28-b9b5-4d4e-9033-c61d7eafaa7b", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:ee:ee", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1404, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "1d2b103e-e328-4cc4-960c-bf1a25d2d121", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.21.0.1", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:22Z", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:30Z", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1d2b103e-e328-4cc4-960c-bf1a25d2d121", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.21.0.0/28", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.21.0.1", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd476201-48c3-4b34-aa8a-442ae669146d", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "857cbe40-ba64-4a8a-a0b0-f7b508dcac1e", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:5e:09:83", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1417, Jul 01 09:31:42.778244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "12a6236e-78c9-4c3d-9ac9-6903ba45fccf", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.22.0.1", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:27Z", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:31Z", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "12a6236e-78c9-4c3d-9ac9-6903ba45fccf", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.22.0.0/28", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.22.0.1", Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id 0a8b585e-b994-4fc1-94b3-5874704a4e13. Wait time elapsed: 6.098 Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id 0a8b585e-b994-4fc1-94b3-5874704a4e13. Wait time elapsed: 6.098 Jul 01 09:31:42.792873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:42.923484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.089s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:42.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:42.926410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:42.926853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:42.927305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:42.929306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c520507d-e0fe-4576-97ff-3fa05230f412]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.952381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10fcb959-82fb-4866-b265-a24f1e1df335]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:42.984598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbf6965a-eb41-41f9-bf0a-ebc895a09966]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.016286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3cd2ef75-7890-4009-97c6-e4b4d2e1a747]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 137998, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.044314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e03ec555-d2b9-4ee8-89b6-eb34229bdb4e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.076930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a10e047b-505f-40b2-8f79-6f8a9adfa8ae]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.120619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[120aac62-2ccc-44b8-876d-5529877affac]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.150116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6db36326-32b0-4dbd-b21b-3d6c91e349bf]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.152330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:43.155091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:43.155603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:43.156028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:43.187121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:43.187834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecbca73f-0d65-4516-bc07-b87001204380]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.193381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3271ea5d-9b37-405d-9d4f-d2fdbafc4e8d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.194608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:43.235100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:43.235815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:43.236194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:43.236605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:43.236980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:43.293121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[312b41ff-5f06-4004-8c56-891e37cfdf10]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.327608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-51400a6b-3065-4fde-84f2-1f6c07bfc05d tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['00025501-514b-4b36-87f4-cf1ff29b34af'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:43.331547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[74ed956f-17c2-437a-98dc-19bb15658aa7]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:43.331547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:43.331547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:43.331547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:43.331547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:43.331547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:43.331547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:43.331547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:43.331547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:43.331547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:43.331547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:43.331547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:43.331547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:43.333114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74ed956f-17c2-437a-98dc-19bb15658aa7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:43.334630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:43.336760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:43.345428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b838f1df-09b3-4cea-a506-a72a52561687]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.346939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:43.347333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:43.347689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:43.348103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:43.349504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[271c4189-b7d1-4bf1-9eb2-47c3c79880de]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.351039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecdf7323-5013-4191-bf6e-7052f4c12f60]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.352050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0edcf07-e32c-4d15-a3b5-03127cd94a61]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.382206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2df4693a-2a54-48df-961b-f738a8da4a89]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 279928, 'reachable_time': 18136, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138016, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.384921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:43.385192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fec6b40-e7a0-4e3c-a11b-e3784cb6c222]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:43.386346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "caef9192-df58-4fac-b455-ffd542999647", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.86", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "70a27689-ef53-4838-bc3d-7d03c0119dc4", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1384, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:68:a6:05", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9ffd697e-81f0-41cc-ab54-952c8a1bbb00", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:11Z", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:43.388981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3 Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id c861a568-988e-40ae-a7bb-0a3190f37a5d. Wait time elapsed: 34.492 Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id c861a568-988e-40ae-a7bb-0a3190f37a5d. Wait time elapsed: 34.492 Jul 01 09:31:43.409791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:44.531807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.143s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:44.533300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:44.536282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:44.541436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:44.541723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:44.543438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aaa12849-ef31-47ba-a9e0-04ac2f00e5cf]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.572776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e52c9e5-69be-4029-b215-8b2c3ecf1a26]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.598596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97aba60c-9c6f-4c46-bc80-dec50d2d8339]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.637259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbef47b6-e5c6-4f92-925b-585f4c42dd1b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 138019, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.660882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a0ebc33-c136-4d35-8a45-898544b313ba]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.699183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1928d8cb-8bd6-45b1-828b-e030a95573aa]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.729746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9eb03644-2a97-497f-b935-7d4506afa64b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.770262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8e2d5e3-356f-4e2e-9780-9601a6e469e2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.771092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:44.771668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:44.772333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:44.772846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:44.797266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:44.797893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d707a61-0dd8-4495-9af7-11d048785b1f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.801454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1067fbd-49c5-42d3-8494-c818cd4df978]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.802790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:44.838592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:44.839922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:44.840617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:44.841687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:44.842306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:44.899852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48789fa6-bcf1-48cf-b017-6d293a41335a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.947633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0003e55a-8129-4e1c-98da-837a376b2896]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:44.947633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:44.947633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:44.947633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:44.947633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:44.947633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:44.947633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:44.947633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:44.947633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:44.947633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:44.947633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:44.947633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:44.947633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:44.951462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.173s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:44.951819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:44.952800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:44.953252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:44.953635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:44.954182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0003e55a-8129-4e1c-98da-837a376b2896]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:44.955736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:44.957169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dab9be59-934f-424a-a3f0-b1ec47b2bf6d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.962372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f562f85-73d2-408b-8c83-df29e34972d7]: (4, ['qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.967244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:44.967787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:44.968186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:44.968694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:44.970648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d77250a5-85e1-48cc-acf7-8d92d7fccad8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.972213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7296a76-3abb-4cbb-908f-e21653fa956f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.979930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f62c56c2-e3eb-490d-bf21-777f787944d4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:44.998570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f16f878-3732-4488-afb5-293f40ffaae0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.024467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02279d35-b919-421e-814f-b4bf868bfc38]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 280090, 'reachable_time': 17308, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138050, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.034691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:45.034941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cbed132-7b8c-4eac-afc4-83d5b4ec6fac]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:45.036140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "caef9192-df58-4fac-b455-ffd542999647", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.86", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "70a27689-ef53-4838-bc3d-7d03c0119dc4", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1384, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:68:a6:05", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9ffd697e-81f0-41cc-ab54-952c8a1bbb00", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:11Z", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:45.040700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:45.053664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:45.058754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id a7fdcc0b-44bf-4b2d-ab67-61ab20b6c71d. Wait time elapsed: 6.419 Jul 01 09:31:45.058754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id a7fdcc0b-44bf-4b2d-ab67-61ab20b6c71d. Wait time elapsed: 6.421 Jul 01 09:31:45.058754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:45.106377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44a02aeb-dbbf-489e-8186-8c21714b6ff0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.180911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dea8dafd-4cc5-4628-b863-3bba5a6e6cd1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 138062, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.220214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83a1fc39-b3ea-4a0c-832d-62e735f8bf6c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.243566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c751aa2-1a49-4eff-b4e2-f095afe185d1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.281694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ac8775f-28e5-4258-8854-255344ae3495]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.326741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7b3fa49-5148-4b45-acec-058e8d464689]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.327724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:45.328234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:45.328744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:45.329196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:45.351658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:45.352149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[647c53db-6afd-4f58-8d1c-544b0c7e3d99]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.356427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7dc2a003-8de2-4317-8ad2-dd347f3a2cfb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.357703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:45.399150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:45.400298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:45.401018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:45.403042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:45.403699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:45.473540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b75a2e9-dde3-4815-aa0a-91cec7bd0aa9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.514325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[db10edd5-f1a7-4d13-a6ca-ee8618362ee4]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:45.514325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:45.514325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:45.514325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:45.514325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:45.514325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:45.514325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:45.514325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:45.514325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:45.514325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:45.514325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:45.514325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:45.514325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:45.521192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db10edd5-f1a7-4d13-a6ca-ee8618362ee4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:45.522851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:45.526100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:45.530526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dea4fbf2-815d-4990-a68d-16cb7d5a6e6c]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.531602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:45.532327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:45.533175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:45.534401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:45.536480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b748d23-0801-4229-8e19-e9ca7829b397]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.538349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bef6a920-a380-4db8-b497-46efafba0f8f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.539472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[477c6d2e-94b3-4336-a4a9-c21a4f25f534]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.573347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5950941-a8f6-4a3d-ad7b-a5640752aae6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 280144, 'reachable_time': 41548, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138081, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.576048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:45.576329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b62ea21-93be-450f-bd5f-4682016a72c9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:31:45.577422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2c5c1a36-bdc1-4847-9f8f-e598677fd30a", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.206", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "5cd44a2d-bdd8-4364-8080-ef931dc139c1", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1347, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:dd:94:6c", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0e6c39b8-7e9c-439c-86ab-2385f592e29b", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:53Z", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:53Z", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:45.581806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:35Z", Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:45.592723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3 Jul 01 09:31:45.952059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.173s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:45.952700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 00025501-514b-4b36-87f4-cf1ff29b34af info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:45.953955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (00025501-514b-4b36-87f4-cf1ff29b34af), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:45.954523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:45.954894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:45.959254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c77bacbf-e526-41ee-bf90-4a608bba90d7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:45.977109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2be6d4b-5a0a-4e8e-8973-c73522bc79f0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.000567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bb2a0c7-5111-4619-ad73-9753da72fd2e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.059120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e675cbd-5a0e-44b3-873c-93004e131730]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 138084, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.095887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c3ecb21-981d-4f14-9a88-d8bb7f297c44]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.124720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f61d016-de6c-487a-856b-330e24afaad5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.146417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19dbbb84-a7b6-45b5-a658-e718d46d014a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.172144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d19c220-6eb3-47ed-882b-e11d575b58d2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.172940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:46.173368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:46.173634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:46.173934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(5797795b-9c38-4a89-aaaf-36f3d27d7977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:46.190076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:46.190332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ebad465-fbb8-46d8-a3f9-39f329880e3e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.193596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8b84fde-884a-432f-9136-aedf2beaccac]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.194704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-5797795b-9c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:46.239019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-5797795b-9c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:46.240331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:46.240938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:46.241333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:46.241711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-5797795b-9c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '5797795b-9c38-4a89-aaaf-36f3d27d7977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6d:28:93'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:46.302978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6163b00-15b2-4b13-b227-4abe49d3b9ed]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.346644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d7948d8e-0e08-4649-985a-c4ac02fa98f4]: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:46.346644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:46.346644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:46.346644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:46.346644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:46.346644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:46.346644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:46.346644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:46.346644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:46.346644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:46.346644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:46.346644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:46.346644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:46.347871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7948d8e-0e08-4649-985a-c4ac02fa98f4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:46.348873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:46.351922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:46.355871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec05ad95-50f1-4a76-8924-aab4a58e2ce9]: (4, ['qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.357018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:46.359991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:46.360485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:46.361305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:46.363103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[583464d0-4792-445c-ac02-d94146df8d8e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.364588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0480545-eba8-44e9-8ae7-5162524a3cac]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.366072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d73cc9da-7f82-4931-b968-ef7c4acbe18a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.391836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a132026-d99a-4fd9-9e03-caa576626e84]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 280230, 'reachable_time': 33104, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138102, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.395177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:46.395624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95b4fe4b-ee10-4056-bd66-53c0d48c3bb1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:46.396806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersTest-router-1948506809", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:15Z", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:42Z", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5797795b-9c38-4a89-aaaf-36f3d27d7977", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "e68c188b-7855-4484-9148-26099a7fbe2e", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6d:28:93", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1393, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:16Z", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:43Z", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88ae0bcc-6d24-4f81-9919-60894e295014", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1321ae28-b9b5-4d4e-9033-c61d7eafaa7b", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:ee:ee", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1404, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "1d2b103e-e328-4cc4-960c-bf1a25d2d121", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.21.0.1", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:22Z", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:30Z", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1d2b103e-e328-4cc4-960c-bf1a25d2d121", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.21.0.0/28", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.21.0.1", Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:46.401143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:46.413026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id df4b42d6-bf90-4994-adfe-559d079e83d6. Wait time elapsed: 7.457 Jul 01 09:31:46.413026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id df4b42d6-bf90-4994-adfe-559d079e83d6. Wait time elapsed: 7.457 Jul 01 09:31:46.413026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:46.834727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ac970b35-49f0-440e-9985-8ed50a87ed4f tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['00025501-514b-4b36-87f4-cf1ff29b34af'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:47.264281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.852s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:47.264555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 00025501-514b-4b36-87f4-cf1ff29b34af info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:47.265888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (00025501-514b-4b36-87f4-cf1ff29b34af), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:47.266366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:47.266812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:47.268104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ef0bc83-a986-4dd4-bfca-74fd130abea8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.285223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba429f89-2e5e-4b09-bd79-125227ca6875]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.306076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[714fbb76-9dfe-4bfe-a65c-39f471c84091]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.337249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00082a23-f42c-41d4-ad60-1d2dc34b710c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 138105, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.358747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0496a9fc-c909-45fe-a2ce-0bd23bf7cfd4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.383589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ab584e2-0ed8-464e-ba7d-ab880a2bc727]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.422444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b30a05d-85e9-4567-b22d-3170d3280925]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.452909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b67e2f38-b8f5-4eea-99e4-5bf3f68255ee]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.453548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:47.453979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:47.454363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:47.454868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(5797795b-9c38-4a89-aaaf-36f3d27d7977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:47.501839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:47.502548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1995665b-aeeb-4334-b90c-11e19349113c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.507494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f52c3900-67af-4ae0-b822-c4b0d4c1d738]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.508819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-5797795b-9c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:47.549470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-5797795b-9c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:47.550489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:47.551140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:47.551758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:47.552371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-5797795b-9c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '5797795b-9c38-4a89-aaaf-36f3d27d7977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6d:28:93'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:47.604372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[507b97fb-0b3c-4d19-8bea-0ebbfc8c91fe]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.643900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a5a90fa7-85b3-4505-ab72-5469a6963bac]: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:47.643900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:47.643900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:47.643900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:47.643900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:47.643900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:47.643900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:47.643900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:47.643900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:47.643900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:47.643900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:47.643900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:47.643900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:47.644740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5a90fa7-85b3-4505-ab72-5469a6963bac]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:47.646366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:47.647905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:47.649805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e407aa1-ce9d-41d5-aa47-41f6922e5121]: (4, ['qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.649805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:47.649805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:47.649805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:47.650087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:47.652211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72299c1d-18c8-4f2f-907f-78691ba50e12]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.653407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e692067f-0a18-49b5-b030-a6fade8d74e7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.654542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[889cbd19-673f-4832-8553-551c19ea5d87]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.680560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa7f83b3-0464-45b6-9221-d2b0c0fd1684]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 280360, 'reachable_time': 34820, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138124, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.689611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:47.689867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[327e17ba-7790-4573-805f-831f9f067988]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:47.691115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersTest-router-1948506809", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:15Z", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:46Z", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5797795b-9c38-4a89-aaaf-36f3d27d7977", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "e68c188b-7855-4484-9148-26099a7fbe2e", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6d:28:93", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1393, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:16Z", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:43Z", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id c2099aa8-d6c2-4d4a-9545-cb495149a734. Wait time elapsed: 4.364 Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id c2099aa8-d6c2-4d4a-9545-cb495149a734. Wait time elapsed: 4.364 Jul 01 09:31:47.693899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:48.089222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.034s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:48.092219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:48.093418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:48.093855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:48.094693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:48.095911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9462772-5418-46e0-a27e-16cec52bb901]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.122093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[519b0396-6323-4990-8172-a607b14c1b61]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.158671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2ca575b-95bf-4b31-a249-2868c95a0614]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.209256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c14fbb9-a2a2-45da-83fb-34ca45cf4942]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 138127, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.241041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[032fdf8f-6736-4d6f-9e9b-9211a8bc15b7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.268996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[223407f3-2134-4b30-b86a-58c1fed312f6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.297926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c31fef5-691e-4323-aedb-69799939fbaf]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.323342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43597413-1d5b-442a-b74f-ece4dce98c3f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.324866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:48.325409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:48.326427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:48.326863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:48.363837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:48.364169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7cb6f4d-e6d5-4d0a-9dc6-8a90dc2a12dd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.373738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f50d0187-a1da-4fb1-af61-02334e1c4805]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.375384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:48.411199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:48.426087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:48.436542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:48.437317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:48.437837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:48.494092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f08b334-95e6-458f-8323-01660f47f610]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.547651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6b59afaa-e104-46a1-b586-8b36152cc4a5]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:48.547651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:48.547651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:48.547651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:48.547651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:48.547651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:48.547651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:48.547651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:48.547651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:48.547651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:48.547651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:48.547651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:48.547651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:48.548601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b59afaa-e104-46a1-b586-8b36152cc4a5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:48.548848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:48.550512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:48.552598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29b8c5cb-dbca-41b1-9bd7-53f811d382ca]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.552598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:48.552598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:48.552598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:48.553627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:48.554202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[471a1037-4258-40b5-af94-79a139f7ad88]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.554889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20fa94d3-d8f5-4de3-a32e-5040286de10e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.557150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f61dc493-1f99-452a-bd08-78a4b52bd10b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.603291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5586a1a9-18f8-4c13-ae67-3fe1910e37f2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 280446, 'reachable_time': 41504, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138145, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.615164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:48.615503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9a5a188-7da1-49c2-b888-a532164708c1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:48.616745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "caef9192-df58-4fac-b455-ffd542999647", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.86", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "70a27689-ef53-4838-bc3d-7d03c0119dc4", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1384, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:68:a6:05", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9ffd697e-81f0-41cc-ab54-952c8a1bbb00", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:11Z", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:48.621422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:46Z", Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id c861a568-988e-40ae-a7bb-0a3190f37a5d. Wait time elapsed: 39.731 Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id c861a568-988e-40ae-a7bb-0a3190f37a5d. Wait time elapsed: 39.738 Jul 01 09:31:48.637096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:49.742969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.108s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:49.743890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:49.744227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:49.744552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:49.744837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:49.747134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f127735c-58a4-4784-9db3-f1392427a9b0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:49.763106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8db7a805-be9e-4bc0-83fe-a1800bb1111d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:49.784793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee4677a4-8008-4261-bc3e-195f40002e96]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:49.820076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12d510e2-c9a7-4a1e-8f56-165a77efd4af]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 138148, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:49.844529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2344e78-dbd4-4dc2-9845-9cafdedfafcf]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:49.866612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2f8ceff-85a4-402b-9a6e-ba73639a962d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:49.890294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2eaf336-b93e-49ce-be74-b44fc5247a8d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:49.914278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b8f057a-7a0a-45dc-a2ef-681488ed46f2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:49.915110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:49.915571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:49.916028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:49.916305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:49.932440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:49.933135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[009b9650-c534-4fea-bbd7-56c2e5b11905]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:49.935942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7c83f6e-c0cd-42c1-8632-188f5bd12226]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:49.936867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:49.973011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:49.973660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:49.975039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:49.975617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:49.976240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:50.040841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76567545-eed2-461f-81ff-d62056282ef1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:50.078954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d6e9f962-cc62-4b79-9e80-21fab1a5d9e6]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:50.078954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:50.078954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:50.078954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:50.078954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:50.078954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:50.078954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:50.078954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:50.078954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:50.078954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:50.078954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:50.078954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:50.078954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:50.080036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6e9f962-cc62-4b79-9e80-21fab1a5d9e6]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:50.080166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:50.082698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:50.084651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d483b9cc-1634-4688-9712-a861efa93715]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:50.084756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:50.085073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:50.085476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:50.085824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:50.086910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1d293de-a1a7-4c01-9914-475c328909e9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:50.087781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d07b0a9-6806-484c-9ed3-790f613359b1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:50.088741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a58a5b5b-956b-4bd3-9182-b1edb99bd9f2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:50.110677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5259e83b-4a1d-4323-bbfc-58075ab70a37]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 280608, 'reachable_time': 25816, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138167, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:50.112735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:50.113215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8598cf4-0053-4bb6-80f6-14f51ab2362a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:50.114379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "caef9192-df58-4fac-b455-ffd542999647", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.86", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "70a27689-ef53-4838-bc3d-7d03c0119dc4", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1384, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:68:a6:05", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9ffd697e-81f0-41cc-ab54-952c8a1bbb00", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:11Z", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:50.119067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:46Z", Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3 Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id a7fdcc0b-44bf-4b2d-ab67-61ab20b6c71d. Wait time elapsed: 11.487 Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id a7fdcc0b-44bf-4b2d-ab67-61ab20b6c71d. Wait time elapsed: 11.487 Jul 01 09:31:50.132841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:50.525011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-d41e0519-d5a2-4c4d-941e-1b558601c77f tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['00025501-514b-4b36-87f4-cf1ff29b34af'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:51.367184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.675s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:51.367822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 00025501-514b-4b36-87f4-cf1ff29b34af info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:51.368423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (00025501-514b-4b36-87f4-cf1ff29b34af), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:51.368997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:51.369277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:51.371193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51d81d97-fc7c-4754-9297-ec2a891db7ac]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.387975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e368626-0335-414c-b7bc-4b244ddd8c2c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.409572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6d9e97d-e1be-4f09-bdd6-6d5408a5454a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.444259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ad57de1-7f4f-41d3-bb00-fcca1ac84797]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 138172, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.459189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ead35677-5ec0-4cdd-a243-9a2effcbb584]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.480883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[718b34d6-fa8a-4f84-a4d1-d7e9d80ac10a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.503159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbe1a6b7-237f-452e-abed-6f990b1abf19]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.523967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64783db4-c3de-49ac-9cbd-636e26cd86e8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.524707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:51.525041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:51.525390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:51.525891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(5797795b-9c38-4a89-aaaf-36f3d27d7977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:51.543451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:51.543752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b99fa45b-c383-4bfa-a2f2-e90e35894ac2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.547073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f74fa323-4100-43b1-9198-3e9fc1974195]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.548123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-5797795b-9c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:51.599911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-5797795b-9c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:51.600583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:51.600972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:51.601522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-5797795b-9c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:51.602054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-5797795b-9c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '5797795b-9c38-4a89-aaaf-36f3d27d7977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6d:28:93'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:51.652365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a5106ff-b835-475f-a806-655c622b05bc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[051586f2-d255-4c58-b5a1-770ef02fe89c]: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:51.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:51.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:51.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:51.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:51.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:51.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:51.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:51.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:51.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:51.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:51.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:51.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:51.713551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[051586f2-d255-4c58-b5a1-770ef02fe89c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:51.714792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:51.717724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46024133-5331-4e26-86b8-f9839aff02e2]: (4, ['qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.722764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:51.722764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:51.722764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:51.722764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:51.722764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b13b0faf-4426-4e1d-abc6-13fc6c9f0800]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.723763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd5e6e72-0ac1-4ed0-8349-4fbf297f12fc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.725349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85da3ae9-f37f-4e7d-8d1e-84d75edfaff9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.754584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3a272a5-28e0-45e8-aec9-0a745c21796a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 280770, 'reachable_time': 33724, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138190, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.766045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:51.766295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2a2f012-9fd1-4c12-b4e5-69c96484515e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 00025501-514b-4b36-87f4-cf1ff29b34af: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-5797795b-9c not found in namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af. Jul 01 09:31:51.767570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-RoutersTest-router-1948506809", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": null, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": null, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:15Z", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:46Z", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": null, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5797795b-9c38-4a89-aaaf-36f3d27d7977", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "e68c188b-7855-4484-9148-26099a7fbe2e", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6d:28:93", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "00025501-514b-4b36-87f4-cf1ff29b34af", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1393, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "4fca58e5-eb4d-43bc-a114-dbb55d416d66", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1" Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:16Z", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:47Z", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "1fb93c5372794be484dc3591b4cb82bd", Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [], Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id e87d7de3-07ad-4632-81c7-529b8082929b. Wait time elapsed: 4.933 Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id e87d7de3-07ad-4632-81c7-529b8082929b. Wait time elapsed: 4.934 Jul 01 09:31:51.770146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:52.208909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.440s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:52.209186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 00025501-514b-4b36-87f4-cf1ff29b34af info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:52.210705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (00025501-514b-4b36-87f4-cf1ff29b34af), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:52.211245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:52.211625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:52.212698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f75cb8d-61b7-493b-a567-13ac64ddcd6c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:52.241588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ad2bc8e-4bb7-4e09-a66e-db040ef44837]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:52.268873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47e0acc8-0747-4dbc-9d96-30946b8f49f0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:52.325566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f54d5b7-4109-45d8-afe2-6dd3a4a05724]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 138193, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:52.353015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3949a5f5-4131-4eac-89f9-f8d24793d2b1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:52.395246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4becd92-b8a1-4fc3-8393-1ac71cecc338]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:52.427777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[352fb5b6-5e73-4919-af1f-7b2e32704956]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:52.463193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf20ec21-96d5-4968-94d4-b0be398ac950]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:52.465963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:52.467820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:52.468405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:52.499580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[661a768e-493e-4983-9d82-80b09067326d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 280858, 'reachable_time': 21764, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138199, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:52.500336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:52.500795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:52.501379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:52.501649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:31:52.524401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58c03eff-2354-4c9c-85c2-d97ac14e4775]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 280858, 'reachable_time': 21764, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138200, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:52.527113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:52.528279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:31:52.533717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-614ce4a9-6968-4df1-83ff-65627e88de4d tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['00025501-514b-4b36-87f4-cf1ff29b34af'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:52.577601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88c46836-3795-4447-b0f5-f74696c3e0d6]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:52.822385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1af0b0cf-7fcf-45f6-961e-70053e67b899]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:52.867812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8558ffdd-27c4-499f-8f2e-6acdbb07990e]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:53.148589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8aa86498-af30-402c-8d2b-313a85919987]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:53.149114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:31:53.149432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:53.149734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:53.150026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" released by "process_external" :: held 0.648s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:31:53.150480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:53.150739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:31:53.152083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:53.152784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:31:53.199786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79e614c3-9853-4aff-963d-62267f0d0478]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:53.286433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47a5862f-9059-4cad-85ea-25298fce5474]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:53.325236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[300c88c5-cd0b-415c-8f97-57481d61e86b]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:53.390232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8384e95e-a4b0-47be-91a3-836c1c4a92b9]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:53.390942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:31:53.391258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:53.391556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:53.391850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" released by "process_address_scope" :: held 0.241s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:31:53.392320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (00025501-514b-4b36-87f4-cf1ff29b34af), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:53.392872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:53.393128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:31:53.393488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:53.393939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:31:53.420253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e9546fd-73b5-41dc-bbf2-e3a8c7b2f7a8]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:53.502234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[746ced9d-2b55-4bab-9bc6-9c9123f3148a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:53.529786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6796e927-8af4-41ad-822f-6da7f99a2da2]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:53 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:31:53 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:53.536549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8a4b0c83-c7c4-4b32-af0c-a8fcf683dee1 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got router deleted notification for 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:31:53.614818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a046907-6fa1-40f2-9271-5d533624df99]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:53.615592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:31:53.616138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:53.617583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:53.618167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-00025501-514b-4b36-87f4-cf1ff29b34af" released by "apply_metadata_nat_rules" :: held 0.225s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:31:53.624034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-00025501-514b-4b36-87f4-cf1ff29b34af Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.haproxy Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 00025501-514b-4b36-87f4-cf1ff29b34af Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:31:53.625414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/00025501-514b-4b36-87f4-cf1ff29b34af.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:31:53.653043 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-00025501-514b-4b36-87f4-cf1ff29b34af[138277]: Proxy listener started. Jul 01 09:31:53.664837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 00025501-514b-4b36-87f4-cf1ff29b34af add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:31:53.665377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 00025501-514b-4b36-87f4-cf1ff29b34af, update_id e87d7de3-07ad-4632-81c7-529b8082929b. Time elapsed: 1.897 Jul 01 09:31:53.665852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id 2d5e41ec-618d-4147-ac1f-3cec1c30fc3c. Wait time elapsed: 1.131 Jul 01 09:31:53.666310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 00025501-514b-4b36-87f4-cf1ff29b34af, action 3, priority 1, update_id 2d5e41ec-618d-4147-ac1f-3cec1c30fc3c. Wait time elapsed: 1.132 Jul 01 09:31:53.666833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:54.225363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-22a68a17-1b5f-48a4-b339-d4a5f7d42c15 tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got routers updated notification :['9c499838-6b9b-420a-810f-b71a532e23dd'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:31:54.226899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9c499838-6b9b-420a-810f-b71a532e23dd, action 3, priority 1, update_id 06ecd3de-7a0d-4b08-8523-88fd273ce4d1. Wait time elapsed: 0.001 Jul 01 09:31:54.227533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9c499838-6b9b-420a-810f-b71a532e23dd, action 3, priority 1, update_id 06ecd3de-7a0d-4b08-8523-88fd273ce4d1. Wait time elapsed: 0.001 Jul 01 09:31:54.228069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:54.282719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.616s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:54.283314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 00025501-514b-4b36-87f4-cf1ff29b34af delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:31:54.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (00025501-514b-4b36-87f4-cf1ff29b34af), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:54.318412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[184c419d-4efd-46af-8996-c17b0f7b0306]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.319977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba1c1e8e-3260-44f2-8d9c-b3f9e88fde2c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.321597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cef48001-6e61-403a-82ff-97182b6ce0da]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.322019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:31:54.323902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[572e4816-e830-4ae5-a9e6-6d00c2ee8bb8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.356738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e90da8a6-83b8-427f-b021-6b874b6a4479]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 280858, 'reachable_time': 21764, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138283, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.357837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:54.358251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:54.391870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[410fa043-8eb1-419c-a98b-f70cf1fbcfe4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 280858, 'reachable_time': 21764, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138284, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.392961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:31:54.393440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:54.393758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/00025501-514b-4b36-87f4-cf1ff29b34af.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:54.394009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:54.394739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 00025501-514b-4b36-87f4-cf1ff29b34af {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:31:54.395772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1c08cee-7bcd-409a-8b1e-aa4cac484d09]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.415410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fbb53fd-eab6-435a-9dde-8c749bf8bc5f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 280858, 'reachable_time': 21764, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138285, 'error': None, 'target': 'qrouter-00025501-514b-4b36-87f4-cf1ff29b34af', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.421657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-00025501-514b-4b36-87f4-cf1ff29b34af deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:54.422470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4802301-b064-4272-9d03-747210aeb711]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.423046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 00025501-514b-4b36-87f4-cf1ff29b34af delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:31:54.423569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (00025501-514b-4b36-87f4-cf1ff29b34af), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:54.424046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:54.424487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:54.424921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:54.425324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:54.425712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 00025501-514b-4b36-87f4-cf1ff29b34af, update_id 2d5e41ec-618d-4147-ac1f-3cec1c30fc3c. Time elapsed: 0.760 Jul 01 09:31:54.671208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.551s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:54.671792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:54.673046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:54.673575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:54.674026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:54.676030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51751545-93d5-4968-95fa-ad83a40884f8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.696034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.468s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:54.696683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9c499838-6b9b-420a-810f-b71a532e23dd info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:54.697870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9c499838-6b9b-420a-810f-b71a532e23dd), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:54.699756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:54.700042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:54.702897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3009075b-b2ae-4e95-8c4a-dadf89ae1087]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.708565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7d28e37-b003-4dfb-aec7-1998dd364802]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.726302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c52a66ba-51c5-4996-a895-6331e74f36ec]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.746646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bbb9dd4-0db3-46d6-b30b-417a372133e5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.752865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61cd280e-4e69-4879-a45d-1d4979cbea6d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.791278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41716b8f-ac49-460d-b02e-23511f335b04]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 138291, 'error': None, 'target': 'qrouter-9c499838-6b9b-420a-810f-b71a532e23dd', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.792663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0f0590e-41d1-46fb-b726-2457f64a4d14]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 138290, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.813323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d351bdc-31ee-4af5-ade2-b825750d7859]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.821189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a686edc2-c38b-418c-81ac-815071dbe8ad]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.827780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[800769af-39bc-4f67-9d6d-d398e445cb26]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.834603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cca39e87-5fd2-45b1-bbd2-f3562fb85c8b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.855559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8d90c48-d236-49c3-9235-4b4da6d86f5b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.861741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d7dd733-9a11-4831-a350-d35dfebf0ffc]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.877064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b212bebe-11ca-4ae7-8ed8-996ddce59e6a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.878203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:54.878959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:54.879615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9c499838-6b9b-420a-810f-b71a532e23dd {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:54.903063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d0806ef-76c1-424e-b473-eed3c4eac9fd]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.904150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:54.904581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:54.904991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:54.905527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:54.914330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d43d3fb-434d-414f-b727-a1447a285364]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 281105, 'reachable_time': 38220, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138304, 'error': None, 'target': 'qrouter-9c499838-6b9b-420a-810f-b71a532e23dd', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.915775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:54.916265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:54.917055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:54.917641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:31:54.929432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:54.930039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66e23b41-c1b8-4837-9d9e-56d9e8a500b8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.937069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a302e38e-cdd9-46f1-a545-42ff21ddbdfb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.940288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:54.966005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:54.969490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:54.970282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:54.971512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:54.971907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:54.981438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14fd31e9-c7c0-4932-9fb5-89dacf9088c9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 281105, 'reachable_time': 38220, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138306, 'error': None, 'target': 'qrouter-9c499838-6b9b-420a-810f-b71a532e23dd', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:54.983186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:54.984275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:31:55.019589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[491b08b6-e302-45cd-a5cd-066990c59299]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.024890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-222d780d-3e7a-4cdc-9a9a-e0d74714244a tempest-RoutersTest-1230765666 tempest-RoutersTest-1230765666-project] Got router deleted notification for 9c499838-6b9b-420a-810f-b71a532e23dd {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:31:55.046904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcb22629-79a3-45f3-bdb4-be271b054816]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.076376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[fbfd2dee-fd8a-4faa-8f7d-d50f604b5e47]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:55.076376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:55.076376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:55.076376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:55.076376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:55.076376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:55.076376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:55.076376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:55.076376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:55.076376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:55.076376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:55.076376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:55.076376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:55.077183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbfd2dee-fd8a-4faa-8f7d-d50f604b5e47]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:55.077675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:55.079242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:55.082541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7aee7581-2334-4c34-a33d-12013d62cb15]: (4, ['qrouter-9c499838-6b9b-420a-810f-b71a532e23dd', 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.082541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:55.082541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:55.082541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:55.082541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:55.083557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56bc158d-bd20-467b-a5e1-8f2932e3cb5d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.091689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd74adbc-51b2-4074-a624-11290d6e34cd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.091689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55544b44-f4dc-47d7-8287-8abcb6c0d756]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.119829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a715233b-f723-43c9-8a3a-e7fbc46099fd]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 281105, 'reachable_time': 26484, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138321, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.140342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:55.140953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc1ede12-e598-43d2-9923-91d1d20fde1c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:55.142190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "caef9192-df58-4fac-b455-ffd542999647", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.86", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "70a27689-ef53-4838-bc3d-7d03c0119dc4", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1384, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:68:a6:05", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9ffd697e-81f0-41cc-ab54-952c8a1bbb00", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:11Z", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:55.146405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:50Z", Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id a7fdcc0b-44bf-4b2d-ab67-61ab20b6c71d. Wait time elapsed: 16.514 Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id a7fdcc0b-44bf-4b2d-ab67-61ab20b6c71d. Wait time elapsed: 16.514 Jul 01 09:31:55.160463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:31:55.282295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00756f82-9a6f-48d7-bc10-c73d6fb51dbf]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.322369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[602ad380-94c2-469e-b389-aa56c5a8d29f]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.530162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2101a9e6-0ab5-4607-bd2e-f8f4fd884485]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.531465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:31:55.531985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:55.532474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:55.532982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" released by "process_external" :: held 0.615s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:31:55.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:55.534089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" acquired by "process_address_scope" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:31:55.534686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:55.535526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:31:55.583479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8b44c83-ca4f-4534-be9e-88b1d1c058c1]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.629671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4c0a3f4-f42a-43f4-aa5f-041682dead2d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.669365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[979672ae-76ce-411e-9b41-cff9e2425e90]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.698131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1caa31ed-2621-43e9-b9b5-8d83995f36a5]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.698748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:31:55.699124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:55.699494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:55.699843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" released by "process_address_scope" :: held 0.166s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:31:55.700281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (9c499838-6b9b-420a-810f-b71a532e23dd), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:55.700819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:55.701168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:31:55.701624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:31:55.702208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:31:55.718074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8cd8698c-6463-448e-9854-06d2d8f5cf7a]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.779098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f91bdb0c-2c74-4c9b-b274-006d81b9a4de]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.792996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[813d5939-11ee-4227-8d57-b43266b568ec]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:31:55 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:31:55 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.874112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06f976e8-8ecb-494d-8d7d-d55fd8e7c7e5]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.874515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:31:55.875005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:55.875450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:31:55.875732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9c499838-6b9b-420a-810f-b71a532e23dd" released by "apply_metadata_nat_rules" :: held 0.175s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:31:55.876234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9c499838-6b9b-420a-810f-b71a532e23dd.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9c499838-6b9b-420a-810f-b71a532e23dd.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-9c499838-6b9b-420a-810f-b71a532e23dd Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/9c499838-6b9b-420a-810f-b71a532e23dd.pid.haproxy Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 9c499838-6b9b-420a-810f-b71a532e23dd Jul 01 09:31:55.876663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:31:55.878238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-9c499838-6b9b-420a-810f-b71a532e23dd', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/9c499838-6b9b-420a-810f-b71a532e23dd.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:31:55.896902 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-9c499838-6b9b-420a-810f-b71a532e23dd[138333]: Proxy listener started. Jul 01 09:31:55.899433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 9c499838-6b9b-420a-810f-b71a532e23dd add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:31:55.899897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 9c499838-6b9b-420a-810f-b71a532e23dd, update_id 06ecd3de-7a0d-4b08-8523-88fd273ce4d1. Time elapsed: 1.673 Jul 01 09:31:55.900327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9c499838-6b9b-420a-810f-b71a532e23dd, action 1, priority 1, update_id df6b44d8-2360-4854-9fc8-53eeb7744be6. Wait time elapsed: 0.875 Jul 01 09:31:55.900574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9c499838-6b9b-420a-810f-b71a532e23dd, action 1, priority 1, update_id df6b44d8-2360-4854-9fc8-53eeb7744be6. Wait time elapsed: 0.875 Jul 01 09:31:55.900865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 9c499838-6b9b-420a-810f-b71a532e23dd delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:31:55.901196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (9c499838-6b9b-420a-810f-b71a532e23dd), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:55.922055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[851b6e84-7d98-483b-b5b2-9308d00320fa]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.924991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f4f8a41-7b46-4d97-b5db-9f7c8db55960]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.926841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[378c8ba9-eb10-462a-bd68-d3dd84d7b160]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.927551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 9c499838-6b9b-420a-810f-b71a532e23dd {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:31:55.929415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28accab9-a866-4981-9ffc-ed44bad97503]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.957255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15dcd38e-d093-4132-992d-fbf4705e5629]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 281105, 'reachable_time': 38220, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138339, 'error': None, 'target': 'qrouter-9c499838-6b9b-420a-810f-b71a532e23dd', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.958451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:55.958883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:55.978742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7b18113-c26d-493a-9a1b-6395f571113e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 281105, 'reachable_time': 38220, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138340, 'error': None, 'target': 'qrouter-9c499838-6b9b-420a-810f-b71a532e23dd', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:55.980258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 9c499838-6b9b-420a-810f-b71a532e23dd {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:31:55.980929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9c499838-6b9b-420a-810f-b71a532e23dd.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9c499838-6b9b-420a-810f-b71a532e23dd.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:55.981440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9c499838-6b9b-420a-810f-b71a532e23dd.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9c499838-6b9b-420a-810f-b71a532e23dd.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:55.981835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 9c499838-6b9b-420a-810f-b71a532e23dd {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:55.982985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 9c499838-6b9b-420a-810f-b71a532e23dd {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:31:55.984824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a332bb4a-2663-4c6d-900c-ae2b1f4215c9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:56.006358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ff58d1e-362d-47f3-bcea-eb59b9617146]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 281105, 'reachable_time': 38220, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138341, 'error': None, 'target': 'qrouter-9c499838-6b9b-420a-810f-b71a532e23dd', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:56.013486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9c499838-6b9b-420a-810f-b71a532e23dd deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:56.013927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e193866c-ee7a-462e-9804-68383b111637]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:56.014373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 9c499838-6b9b-420a-810f-b71a532e23dd delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:31:56.014764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (9c499838-6b9b-420a-810f-b71a532e23dd), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:56.015114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:56.015423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:56.015801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:56.016074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:56.016326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9c499838-6b9b-420a-810f-b71a532e23dd, update_id df6b44d8-2360-4854-9fc8-53eeb7744be6. Time elapsed: 0.116 Jul 01 09:31:58.425359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.277s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:31:58.425962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:31:58.427089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:31:58.427768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:58.428291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:58.429710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49660391-1740-4c56-b5f3-6147595bb31f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.470147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32be3e37-c037-4632-872d-685ff925e7df]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.513711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88961354-7f07-4c15-b5bb-32d0b737e277]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.576439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2940f7d3-1c78-4e9b-972e-8669a15410f1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 138370, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.642336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6de21738-814d-4966-abef-015b4133eb09]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.683582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[880c40f0-e4da-43c7-8c8f-405d849c058f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.724876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ed9d79e-cd3b-4f4b-a90c-79b183029c45]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.789739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0206cb32-22c5-4cc0-bdfc-37376be95c19]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.790280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:31:58.792256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:31:58.792699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:31:58.793493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:31:58.826077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:31:58.826366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f25acab4-5daa-4806-b270-c1d5c5955747]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.829701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2021bd31-95f8-485c-b1d3-005945b52954]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.830642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:58.865353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:58.866438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:58.867055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:58.867632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:58.868210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:31:58.914983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfcfc24a-cb00-41e1-80d6-460ed15401c2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cda9c5ce-26b8-4566-9aa7-4ec295f358b8]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:31:58.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:31:58.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:31:58.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:31:58.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:31:58.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:31:58.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:31:58.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:31:58.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:31:58.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:31:58.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:31:58.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:31:58.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:58.977019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cda9c5ce-26b8-4566-9aa7-4ec295f358b8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:58.978162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:58.981482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:58.985716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e389d16-09ef-4ecf-b915-88eef5d32e76]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.986750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:58.987241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:58.987706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:31:58.988198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:31:58.989320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76bfc531-4d85-4edd-a3ff-9c54bf363786]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.990486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76d6259b-138e-4c6c-87af-0a8a79a8fb28]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:58.992335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cfc371f-b06a-46fe-b648-8aa20389ee90]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:59.029412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[763a037b-0739-4b38-8b7f-cf52da434e62]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 281484, 'reachable_time': 41480, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138402, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:59.038564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:31:59.038872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75eb3a00-114a-4981-a0d0-bfe5a013d002]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:31:59.039976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "caef9192-df58-4fac-b455-ffd542999647", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.86", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "70a27689-ef53-4838-bc3d-7d03c0119dc4", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1384, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:68:a6:05", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9ffd697e-81f0-41cc-ab54-952c8a1bbb00", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:11Z", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:31:59.044581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:55Z", Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id a7fdcc0b-44bf-4b2d-ab67-61ab20b6c71d. Wait time elapsed: 20.423 Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id a7fdcc0b-44bf-4b2d-ab67-61ab20b6c71d. Wait time elapsed: 20.424 Jul 01 09:31:59.062122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:32:00.150960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.094s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:32:00.151737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:32:00.153048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:32:00.153386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:32:00.153801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:32:00.155603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4474273-4af0-4c52-817a-dfdf0eea352e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.166437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84bcdb8b-6f8a-48fb-b0c1-dff958a60506]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.189601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7de7814-6fae-4bdd-8efe-2ad82dd9ec35]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.225569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[569b2ffe-8240-4c67-975d-68ffbbef34ad]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 138408, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.247978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f28115f8-9987-41e1-804d-b633ea0021f2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.271291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8479750-1727-4f9f-979f-1098da34209a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.293689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78461fed-82de-4c91-b451-473ba5af3960]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.319092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9ad9197-967b-460d-9c72-4fbce31ea34f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.320160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:32:00.320631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:32:00.320954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:32:00.321388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:32:00.340256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:32:00.340512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d5eac21-16bc-4d2b-ab47-5da1d60ef9f5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.346357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83cc2987-eabe-4018-aef1-198ff2a578f2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.347633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:32:00.375623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:32:00.376581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:32:00.377252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:32:00.377812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:32:00.378383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:32:00.447129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6279010c-9867-4c7b-9064-e377096a8eaf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.504393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7a957b60-c25f-4ea7-930e-854d545d671a]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:32:00.504393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:32:00.504393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:32:00.504393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:32:00.504393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:32:00.504393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:32:00.504393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:32:00.504393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:32:00.504393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:32:00.504393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:32:00.504393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:32:00.504393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:32:00.504393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:00.505282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a957b60-c25f-4ea7-930e-854d545d671a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:00.505494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:00.507183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:32:00.509235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31a87101-7656-404a-8f14-589490228bcb]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.509235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:32:00.509235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:32:00.509235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:32:00.509235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:32:00.510024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[261f507e-eacb-4d03-a680-da715efa3ff1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.510971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47b5e19c-7353-4c42-b000-a74e70e6a06e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.511897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[330979ff-78b2-4ab7-b61e-69a321b274e4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.533100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eff49c12-abeb-4188-bf8c-baadd2c12144]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 281648, 'reachable_time': 38936, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138427, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.540326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:32:00.540603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab3ead94-2fdf-4820-95da-e9f0178142f6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:00.541694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "caef9192-df58-4fac-b455-ffd542999647", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.86", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "70a27689-ef53-4838-bc3d-7d03c0119dc4", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1384, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:68:a6:05", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9ffd697e-81f0-41cc-ab54-952c8a1bbb00", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:11Z", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:32:00.544913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:57Z", Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id a7fdcc0b-44bf-4b2d-ab67-61ab20b6c71d. Wait time elapsed: 21.910 Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id a7fdcc0b-44bf-4b2d-ab67-61ab20b6c71d. Wait time elapsed: 21.911 Jul 01 09:32:00.554765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:32:04.489528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.939s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:32:04.490208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:32:04.497158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:32:04.497653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:32:04.500714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:32:04.517222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e60b3a6c-a692-4a12-9384-037f0d00abe4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.548500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27ebc541-9d57-4b46-9414-9af1dc85975d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.576807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b177dbf-e116-401a-b53a-ac79c82212a2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.626152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6809c888-a357-455b-808e-2a5436ec45b0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 138528, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.656444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a44036df-fd66-4897-a0ea-eb048cacde4d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.686168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ce8ec5c-ff3e-4f23-9d9d-2c6712a639d0]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.718120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91710c6a-eac2-48d2-9dc3-486af2c911b2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.745309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d330263f-171f-4e64-b1dc-f7bf9d657b15]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.747039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:32:04.747582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:32:04.748359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:32:04.748908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:32:04.774331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:32:04.774949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8180b6b1-22cd-45b8-9eea-17764e056853]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.780227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbcf24fd-faa4-4c48-99d4-1d3b68588333]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.781589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:32:04.824843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:32:04.825900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:32:04.826533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:32:04.827202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:32:04.828763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:32:04.899172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db30033e-5c0d-4e7f-b193-aaf303eb8867]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.937344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[33db2fd3-b251-4bf3-a86f-963f0e60526d]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:32:04.937344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:32:04.937344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:32:04.937344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:32:04.937344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:32:04.937344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:32:04.937344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:32:04.937344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:32:04.937344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:32:04.937344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:32:04.937344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:32:04.937344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:32:04.937344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:04.938212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33db2fd3-b251-4bf3-a86f-963f0e60526d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:04.938579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:04.940116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:32:04.942030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd426cf7-c122-4811-a17d-61c57a47115d]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.942030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:32:04.943790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:32:04.943790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:32:04.944136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:32:04.945583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c8bacb0-89fb-4b3c-ac97-f591fff1b147]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.946552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9b065a9-11b4-4524-86df-02141cb701a8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.948568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cae1a1f2-93b6-40c2-b3cb-e964360bcf28]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.976004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5523d403-2468-4a05-9e31-be222fe60b7a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 282088, 'reachable_time': 26504, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 138548, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.982497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:32:04.982759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e24aaf3b-f4a6-4923-a14a-e013eb69cb40]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:32:04.983842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "caef9192-df58-4fac-b455-ffd542999647", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.86", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "70a27689-ef53-4838-bc3d-7d03c0119dc4", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1384, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:68:a6:05", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9ffd697e-81f0-41cc-ab54-952c8a1bbb00", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:11Z", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:37Z", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:32:04.986347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:32:01Z", Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:32:04.997098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3 Jul 01 09:32:15.078163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:32:15.078946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:32:15.078946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:32:15.079158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:32:15.107539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:32:23.249537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8d19d590-9b3c-480b-84e2-b0b9e9038afa tempest-PortsIpV6TestJSON-1482396560 tempest-PortsIpV6TestJSON-1482396560-project] Got routers updated notification :['50607977-f521-442c-a644-6f67786257a1'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:32:23.254399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 50607977-f521-442c-a644-6f67786257a1, action 3, priority 1, update_id 2f6c8ba4-6f51-4a49-8561-d0527de2ff85. Wait time elapsed: 0.001 Jul 01 09:32:23.254653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 50607977-f521-442c-a644-6f67786257a1, action 3, priority 1, update_id 2f6c8ba4-6f51-4a49-8561-d0527de2ff85. Wait time elapsed: 0.001 Jul 01 09:32:23.254952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:32:23.352681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.096s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:32:23.352945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 50607977-f521-442c-a644-6f67786257a1 was not found. Performing router cleanup Jul 01 09:32:23.354870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c046b1a-671d-441c-8cc4-ea850d1d8be6]: (4, ['qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:23.355503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:32:23.355951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:32:23.356366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 50607977-f521-442c-a644-6f67786257a1, update_id 2f6c8ba4-6f51-4a49-8561-d0527de2ff85. Time elapsed: 0.102 Jul 01 09:32:24.480406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-71af58f6-677c-4d6f-ae64-d161f9b5bed5 tempest-PortsIpV6TestJSON-1482396560 tempest-PortsIpV6TestJSON-1482396560-project] Got routers updated notification :['50607977-f521-442c-a644-6f67786257a1'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:32:24.485140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 50607977-f521-442c-a644-6f67786257a1, action 3, priority 1, update_id 47da0043-7452-4217-8369-ba943c97f133. Wait time elapsed: 0.004 Jul 01 09:32:24.485419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 50607977-f521-442c-a644-6f67786257a1, action 3, priority 1, update_id 47da0043-7452-4217-8369-ba943c97f133. Wait time elapsed: 0.005 Jul 01 09:32:24.485836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:32:24.819415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-9169233f-9321-4e57-b172-e05e54b1de16 tempest-PortsIpV6TestJSON-1482396560 tempest-PortsIpV6TestJSON-1482396560-project] Got router deleted notification for 50607977-f521-442c-a644-6f67786257a1 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:32:24.916758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.431s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:32:24.917459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 50607977-f521-442c-a644-6f67786257a1 was not found. Performing router cleanup Jul 01 09:32:24.918902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7c577d6-223f-44c3-a4f9-950ae5de6e8a]: (4, ['qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:24.922300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:32:24.922685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:32:24.923013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 50607977-f521-442c-a644-6f67786257a1, update_id 47da0043-7452-4217-8369-ba943c97f133. Time elapsed: 0.438 Jul 01 09:32:24.923560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 50607977-f521-442c-a644-6f67786257a1, action 1, priority 1, update_id d9064059-6d20-4a69-8139-2013f52d9916. Wait time elapsed: 0.103 Jul 01 09:32:24.923560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 50607977-f521-442c-a644-6f67786257a1, action 1, priority 1, update_id d9064059-6d20-4a69-8139-2013f52d9916. Wait time elapsed: 0.104 Jul 01 09:32:24.924441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 50607977-f521-442c-a644-6f67786257a1 was not found. Performing router cleanup Jul 01 09:32:24.926552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a9e60c3-8c58-40c9-85bf-3058917e5526]: (4, ['qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:32:24.927157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:32:24.928379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:32:24.928689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 50607977-f521-442c-a644-6f67786257a1, update_id d9064059-6d20-4a69-8139-2013f52d9916. Time elapsed: 0.005 Jul 01 09:32:55.109439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:33:15.078222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:33:15.078903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:33:15.079486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:33:15.079979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:33:35.110976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:34:07.425834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-294c0e65-2e16-4c25-89dc-e869dae6b306 tempest-ServerActionsTestJSON-77649674 tempest-ServerActionsTestJSON-77649674-project] Got routers updated notification :['9974ba83-7306-4b67-8057-d6ab904a1c00'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:34:07.437190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 1, update_id 891e22ab-7ec2-4ff9-9c6c-4c49c6a770c7. Wait time elapsed: 0.010 Jul 01 09:34:07.437624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 1, update_id 891e22ab-7ec2-4ff9-9c6c-4c49c6a770c7. Wait time elapsed: 0.010 Jul 01 09:34:07.438078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:34:08.581978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.144s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:34:08.582632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:34:08.583189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:34:08.583548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:08.584403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:08.585669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d1b5677-15d5-48c7-9ba4-a411d0f3725f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.597165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ac149a7-4cc2-474a-9699-22acd0400c50]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.616997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00f8bf95-1752-40c8-b8c7-aa53dc46061d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.642683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[859b8a3b-9a3a-43b6-a28d-c6926a5ba90c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 140538, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.663488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c3f4390-a893-4f8f-bc5f-760c68bf3a14]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.683124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4907648c-1af5-4ccb-9c3a-a78553ba4b99]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.707941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9c0257f-41b8-4a31-bed1-4aa13faf287f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.729464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9851db7b-0415-4c77-8e0b-b17bcf987fe2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.730234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:08.730571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:08.730898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:34:08.731241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:34:08.745854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:34:08.746047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c4ca385-8fcf-4a0a-99f8-c7fcc2f30eba]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.748732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a201e746-55b7-45f0-b5f2-fec2c406700b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.750178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:34:08.777733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:34:08.778957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:34:08.779738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:34:08.780345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:34:08.781114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:34:08.827285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b263a402-8af4-46a0-840b-edb759059488]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.852132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4eb50d4e-e2e3-4bb9-acb6-56e70f0ee287]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:34:08.852132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:34:08.852132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:34:08.852132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:34:08.852132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:34:08.852132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:34:08.852132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:34:08.852132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:34:08.852132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:34:08.852132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:34:08.852132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:34:08.852132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:34:08.852132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:08.853212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4eb50d4e-e2e3-4bb9-acb6-56e70f0ee287]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:08.853298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:08.855184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:34:08.857744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84be1495-2efe-46b0-b496-eedaf99ed04a]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.857874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:34:08.858220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:34:08.858472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:34:08.858879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:34:08.859979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f16504f0-3f1e-44af-b717-507cff573c00]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.860888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a83417f-897e-4333-89cf-9a85f51e4035]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.861807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2154cb66-2e2b-49ca-8c36-a99e2476dddb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.881128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37833833-2d79-44be-89f3-1a15dbbb4875]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 294490, 'reachable_time': 42728, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 140556, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.887079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:34:08.887510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dcf4b6dc-bb89-41e3-87aa-b0133740f24b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:08.888643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:35Z", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:34:08.893459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 891e22ab-7ec2-4ff9-9c6c-4c49c6a770c7. Wait time elapsed: 1.463 Jul 01 09:34:08.905432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 2, update_id 891e22ab-7ec2-4ff9-9c6c-4c49c6a770c7. Wait time elapsed: 1.464 Jul 01 09:34:08.905432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:34:11.479962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-27bd53f8-f3f4-4d08-8681-c22c4e0e2aec admin admin] Got routers updated notification :['9974ba83-7306-4b67-8057-d6ab904a1c00'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:34:11.643860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.752s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:34:11.644342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:34:11.645268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:34:11.645715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:11.645999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:11.648104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8fbe171-254c-475d-a8dc-41c1394efddf]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:11.661021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01d32014-5e86-4c57-aa91-3b3b01e674f2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:11.685149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8deaec6-9798-4d3b-91c2-cd37b0039d29]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:11.718085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9fb81bb8-1871-444d-ad1c-6f96177c4b61]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 140561, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:11.740714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b96c3b31-2daf-4994-b203-8cdc075059bc]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:11.772452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cc048de-38f3-47a8-9747-ec4095acc4c7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:11.804463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[467bd18f-44b7-47bf-bdc9-4bd6ff9cf308]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:11.826526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99068334-85de-45a8-9657-9e248efb88a9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:11.827585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:11.828123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:11.828718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:34:11.829243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(6734468f-b992-441c-9861-055ec2d4f564) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:34:11.848461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:34:11.850199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[852de253-e90a-47cd-8fa7-32094615653d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:11.855553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13f643e8-2dab-46af-93b5-e82da390b18d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:11.856725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-6734468f-b9, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:34:11.902939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-6734468f-b9, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:34:11.903823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:34:11.904387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:34:11.904768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-6734468f-b9, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:34:11.905150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-6734468f-b9, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6734468f-b992-441c-9861-055ec2d4f564', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:20:44:69'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:34:11.966380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53355d8b-9914-4a77-b046-4352d17af035]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.006502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ddda2ae2-6e47-462f-8ede-faabce526dea]: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:34:12.006502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:34:12.006502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:34:12.006502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:34:12.006502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:34:12.006502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:34:12.006502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:34:12.006502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:34:12.006502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:34:12.006502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:34:12.006502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:34:12.006502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:34:12.006502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:12.007577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddda2ae2-6e47-462f-8ede-faabce526dea]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:12.007791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:12.009545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:34:12.011838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d47ee2e0-412f-4a3a-aa1b-513eae61a765]: (4, ['qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.011838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:34:12.011838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:34:12.011838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:34:12.011838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:34:12.014016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bc6435d-f751-4eb1-819c-f52a51489072]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.014949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d8dbcb6-893b-49a6-aa1f-6c87e1c923f9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.016273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1fb0500-8d6e-4245-92f2-5921134416f9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.035250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6518fdb3-03d4-43d7-b6f2-7432359ce938]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 294798, 'reachable_time': 35640, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 140582, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.038060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:34:12.038506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b488b93-5900-487f-824d-267a33f6e9fe]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9974ba83-7306-4b67-8057-d6ab904a1c00: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-6734468f-b9 not found in namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00. Jul 01 09:34:12.039675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-543597614-router", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132" Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:53Z", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:04Z", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fdf456c3-4b9b-4ddb-95a4-0e9aa788e001", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:56:61", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 412, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.132", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:25:54Z", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:26:11Z", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6734468f-b992-441c-9861-055ec2d4f564", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:20:44:69", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9974ba83-7306-4b67-8057-d6ab904a1c00", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 466, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:26:01Z", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:27:35Z", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "ca1c1fe6189c41a58e0af042d8aadb8f", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f686b082-ad1a-4676-aefe-635ceef2565c", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:34:12.045065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:34:12.058625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 1, update_id e892ace3-da09-442d-b165-e9f0dfee2f3c. Wait time elapsed: 0.576 Jul 01 09:34:12.058625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 3, priority 1, update_id e892ace3-da09-442d-b165-e9f0dfee2f3c. Wait time elapsed: 0.576 Jul 01 09:34:12.058625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:34:12.529049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-51aad6ce-29a9-4085-9d7f-c47d49100a84 admin admin] Got router deleted notification for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:34:12.538916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.482s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:34:12.539481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9974ba83-7306-4b67-8057-d6ab904a1c00 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:34:12.540709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:34:12.541474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:12.541983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:12.544340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[240b5cdc-c2a3-4a35-a3e3-c04fae66b07d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.559576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53c8bca2-ff53-4a8e-b199-f88c4ae4fbcd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.581684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df2dd87b-799e-475e-a5d9-a469c128caa2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.610606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d82b04c-745b-4534-907c-a82c08430c3a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 140612, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.633196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7dd2b8cb-4b09-4680-a093-89ce91865f46]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.657762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e217bb91-2221-4d50-adc8-67e378fbda26]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.680671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab24947e-92e7-4d9c-8482-07b0fd990cb0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.697018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8277fb9-fcfb-44a3-998a-cfbef8a62272]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.697565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:12.697888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:12.698491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:34:12.719718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d057dad5-0da9-4c31-ac7d-114f3a3f3d22]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 294887, 'reachable_time': 33048, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 140618, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.720717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:12.721339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:12.722555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:34:12.722555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:34:12.747853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a812918-9796-4ce6-aab1-a823f3774ecd]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 294887, 'reachable_time': 33048, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 140619, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.749156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:34:12.750010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:34:12.761741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d73bf5f7-dd0f-42c7-a96b-4c5740bc8372]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.934140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b78550ef-692b-44fd-ba66-365f774c8d7a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:12.945746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca95b621-9406-4275-b63b-20454b2d1a3c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.107116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dea37cbf-90fe-4357-b525-1ea6a6aaa772]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.107806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:34:13.108345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:34:13.110024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:34:13.110781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" released by "process_external" :: held 0.388s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:34:13.112325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:34:13.112716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:34:13.113180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:34:13.113795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:34:13.149775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1287b7fb-4bc5-48a6-8439-38c034776080]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.205607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38c7e8dd-395d-48f4-afc9-26cbff54c65e]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.232385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d480b547-d159-41a2-9834-e30c1e3daddc]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.274848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01fc41e1-2632-4321-8fe5-df1721cd5e94]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.274952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:34:13.275261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:34:13.275559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:34:13.275865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" released by "process_address_scope" :: held 0.163s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:34:13.276233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:34:13.276885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:34:13.277209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:34:13.277529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:34:13.277991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:34:13.297639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddba45fd-4ded-456c-9267-9ceb502b6f9d]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.362940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72a599c3-f86b-49bf-abe8-67696f1f168a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.377086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70c3a250-c0cb-4e10-bba0-b3f9edbf2fc4]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:34:13 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:34:13 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.442320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fe866c1-940a-4e14-b618-49bbe9012740]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.443964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:34:13.444577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:34:13.444865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:34:13.445345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00" released by "apply_metadata_nat_rules" :: held 0.168s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:34:13.446333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-9974ba83-7306-4b67-8057-d6ab904a1c00 Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.haproxy Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 9974ba83-7306-4b67-8057-d6ab904a1c00 Jul 01 09:34:13.446899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:34:13.448688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/9974ba83-7306-4b67-8057-d6ab904a1c00.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:34:13.468756 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-9974ba83-7306-4b67-8057-d6ab904a1c00[140633]: Proxy listener started. Jul 01 09:34:13.472929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 9974ba83-7306-4b67-8057-d6ab904a1c00 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:34:13.473533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, update_id e892ace3-da09-442d-b165-e9f0dfee2f3c. Time elapsed: 1.418 Jul 01 09:34:13.474053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9974ba83-7306-4b67-8057-d6ab904a1c00, action 1, priority 1, update_id cbefc845-06fe-4e51-a0b9-dedf5498154d. Wait time elapsed: 0.944 Jul 01 09:34:13.475311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9974ba83-7306-4b67-8057-d6ab904a1c00, action 1, priority 1, update_id cbefc845-06fe-4e51-a0b9-dedf5498154d. Wait time elapsed: 0.944 Jul 01 09:34:13.476125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 9974ba83-7306-4b67-8057-d6ab904a1c00 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:34:13.476676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:34:13.508469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66b2d2b0-52db-4b6a-80c9-57fb4c6d5618]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.510959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44c59f5b-097f-472c-9933-eba343e3eabb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.512163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0e5759c-ff31-4c43-8d0a-252e4186c506]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.512584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:34:13.513927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ccfc6ba0-0206-48b7-a19f-d7e3523463f5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.533337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d39946d1-c11b-4dff-94d0-b8c09bc8edb8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 294887, 'reachable_time': 33048, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 140639, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.534235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:13.534515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:13.564412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39779557-6400-4dd9-b889-8d66387a5748]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 294887, 'reachable_time': 33048, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 140640, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.565538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:34:13.566316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:34:13.566435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9974ba83-7306-4b67-8057-d6ab904a1c00.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:34:13.566806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:34:13.568701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 9974ba83-7306-4b67-8057-d6ab904a1c00 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:34:13.570807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4027ffe1-7d98-49b7-9ebc-f7485d8d87ff]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.595138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca21e94f-87b7-4770-8909-179ebd34f20b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 294887, 'reachable_time': 33048, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 140641, 'error': None, 'target': 'qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.601232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9974ba83-7306-4b67-8057-d6ab904a1c00 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:34:13.601550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8aa39d4b-3ec6-4126-8930-08366553d84c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:13.602069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 9974ba83-7306-4b67-8057-d6ab904a1c00 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:34:13.602462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (9974ba83-7306-4b67-8057-d6ab904a1c00), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:34:13.603427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:13.603772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:13.604157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:13.604486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:13.604739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9974ba83-7306-4b67-8057-d6ab904a1c00, update_id cbefc845-06fe-4e51-a0b9-dedf5498154d. Time elapsed: 0.131 Jul 01 09:34:15.080917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:15.081608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:15.081608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:15.081803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:15.112094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:34:19.310385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-e32fb899-dec8-4767-9516-f10d90136723 tempest-FloatingIPAdminTestJSON-222952656 tempest-FloatingIPAdminTestJSON-222952656-project] Got routers updated notification :['9c69a785-66f7-455c-8e14-1c54e2afef5c'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:34:19.315939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9c69a785-66f7-455c-8e14-1c54e2afef5c, action 3, priority 1, update_id 7cc2fb3c-2825-4315-94de-652382fb8aa8. Wait time elapsed: 0.004 Jul 01 09:34:19.318495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9c69a785-66f7-455c-8e14-1c54e2afef5c, action 3, priority 1, update_id 7cc2fb3c-2825-4315-94de-652382fb8aa8. Wait time elapsed: 0.005 Jul 01 09:34:19.319674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:34:19.874053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.554s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:34:19.874501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9c69a785-66f7-455c-8e14-1c54e2afef5c was not found. Performing router cleanup Jul 01 09:34:19.876024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43280e0f-ac88-4283-b4ef-ab44a077599a]: (4, ['qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:19.876776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:19.877055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:19.877450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9c69a785-66f7-455c-8e14-1c54e2afef5c, update_id 7cc2fb3c-2825-4315-94de-652382fb8aa8. Time elapsed: 0.563 Jul 01 09:34:21.680825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-05df704a-70c7-4bed-a9f9-5a4f5c4ffc31 tempest-FloatingIPAdminTestJSON-222952656 tempest-FloatingIPAdminTestJSON-222952656-project] Got routers updated notification :['9c69a785-66f7-455c-8e14-1c54e2afef5c'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:34:21.684638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9c69a785-66f7-455c-8e14-1c54e2afef5c, action 3, priority 1, update_id 2c28f585-90e6-4114-a0cf-311c432c9f6e. Wait time elapsed: 0.003 Jul 01 09:34:21.684885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9c69a785-66f7-455c-8e14-1c54e2afef5c, action 3, priority 1, update_id 2c28f585-90e6-4114-a0cf-311c432c9f6e. Wait time elapsed: 0.004 Jul 01 09:34:21.685350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:34:21.814325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.129s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:34:21.814648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9c69a785-66f7-455c-8e14-1c54e2afef5c was not found. Performing router cleanup Jul 01 09:34:21.816632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82e9db20-627c-4929-8ee9-e35c5822f8e5]: (4, ['qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:21.817530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:21.818945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:21.818945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9c69a785-66f7-455c-8e14-1c54e2afef5c, update_id 2c28f585-90e6-4114-a0cf-311c432c9f6e. Time elapsed: 0.134 Jul 01 09:34:24.893878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-0cf89176-01f6-4d08-b106-16a40310f925 tempest-FloatingIPAdminTestJSON-343612432 tempest-FloatingIPAdminTestJSON-343612432-project-admin] Got routers updated notification :['9c69a785-66f7-455c-8e14-1c54e2afef5c'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:34:24.895432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9c69a785-66f7-455c-8e14-1c54e2afef5c, action 3, priority 1, update_id 2fa20e0a-0b99-4c46-b4f8-1da2b8799a58. Wait time elapsed: 0.001 Jul 01 09:34:24.896357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9c69a785-66f7-455c-8e14-1c54e2afef5c, action 3, priority 1, update_id 2fa20e0a-0b99-4c46-b4f8-1da2b8799a58. Wait time elapsed: 0.001 Jul 01 09:34:24.896814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:34:25.069012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.172s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:34:25.069456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9c69a785-66f7-455c-8e14-1c54e2afef5c was not found. Performing router cleanup Jul 01 09:34:25.071028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b63f7e9-5415-4f7d-ad6c-6803804d9e0c]: (4, ['qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:25.072284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:25.072810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:25.073249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9c69a785-66f7-455c-8e14-1c54e2afef5c, update_id 2fa20e0a-0b99-4c46-b4f8-1da2b8799a58. Time elapsed: 0.178 Jul 01 09:34:26.045945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-1b5e87d5-aeec-4da1-be82-ab0253c5a588 tempest-FloatingIPAdminTestJSON-222952656 tempest-FloatingIPAdminTestJSON-222952656-project] Got routers updated notification :['9c69a785-66f7-455c-8e14-1c54e2afef5c'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:34:26.047113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9c69a785-66f7-455c-8e14-1c54e2afef5c, action 3, priority 1, update_id 77445d74-4c66-4d03-aed6-85b3037a4743. Wait time elapsed: 0.001 Jul 01 09:34:26.047659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9c69a785-66f7-455c-8e14-1c54e2afef5c, action 3, priority 1, update_id 77445d74-4c66-4d03-aed6-85b3037a4743. Wait time elapsed: 0.001 Jul 01 09:34:26.048170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:34:26.164537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.116s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:34:26.164797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9c69a785-66f7-455c-8e14-1c54e2afef5c was not found. Performing router cleanup Jul 01 09:34:26.166018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36d0884f-78eb-4dbd-bba0-415c0820d1f1]: (4, ['qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:26.166870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:26.167346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:26.167837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9c69a785-66f7-455c-8e14-1c54e2afef5c, update_id 77445d74-4c66-4d03-aed6-85b3037a4743. Time elapsed: 0.121 Jul 01 09:34:31.501785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-243b83f0-0d62-4165-b4c2-0be5eecd804b tempest-FloatingIPAdminTestJSON-222952656 tempest-FloatingIPAdminTestJSON-222952656-project] Got routers updated notification :['9c69a785-66f7-455c-8e14-1c54e2afef5c'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:34:31.502685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9c69a785-66f7-455c-8e14-1c54e2afef5c, action 3, priority 1, update_id de6b320f-b320-4f7b-a523-f6dcbd7cb22f. Wait time elapsed: 0.001 Jul 01 09:34:31.502931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9c69a785-66f7-455c-8e14-1c54e2afef5c, action 3, priority 1, update_id de6b320f-b320-4f7b-a523-f6dcbd7cb22f. Wait time elapsed: 0.001 Jul 01 09:34:31.503275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:34:31.674447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.171s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:34:31.674712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9c69a785-66f7-455c-8e14-1c54e2afef5c was not found. Performing router cleanup Jul 01 09:34:31.675865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2183399-1b2c-4c4a-9d02-60ce5f9669c6]: (4, ['qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:31.677328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:31.677669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:31.677865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9c69a785-66f7-455c-8e14-1c54e2afef5c, update_id de6b320f-b320-4f7b-a523-f6dcbd7cb22f. Time elapsed: 0.175 Jul 01 09:34:32.268147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-2826b8bb-7e61-404e-b822-11dcb3a287dc tempest-FloatingIPAdminTestJSON-222952656 tempest-FloatingIPAdminTestJSON-222952656-project] Got router deleted notification for 9c69a785-66f7-455c-8e14-1c54e2afef5c {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:34:32.269720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9c69a785-66f7-455c-8e14-1c54e2afef5c, action 1, priority 1, update_id a109dd61-3085-496c-9e35-a441b91cedbe. Wait time elapsed: 0.001 Jul 01 09:34:32.270130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9c69a785-66f7-455c-8e14-1c54e2afef5c, action 1, priority 1, update_id a109dd61-3085-496c-9e35-a441b91cedbe. Wait time elapsed: 0.001 Jul 01 09:34:32.271012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9c69a785-66f7-455c-8e14-1c54e2afef5c was not found. Performing router cleanup Jul 01 09:34:32.273561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abcd58fa-2086-493d-bed9-b1b9c4607e85]: (4, ['qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:34:32.274947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:34:32.275600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:34:32.277107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9c69a785-66f7-455c-8e14-1c54e2afef5c, update_id a109dd61-3085-496c-9e35-a441b91cedbe. Time elapsed: 0.006 Jul 01 09:34:55.114738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:35:11.888021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-88995e5a-5fae-4668-a446-619dffd835c3 tempest-ServersTestManualDisk-1149109101 tempest-ServersTestManualDisk-1149109101-project] Got routers updated notification :['22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:35:11.889539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 1, update_id 7d6474b5-6182-401d-aa2e-d525cd468441. Wait time elapsed: 0.001 Jul 01 09:35:11.890040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 1, update_id 7d6474b5-6182-401d-aa2e-d525cd468441. Wait time elapsed: 0.002 Jul 01 09:35:11.890553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:35:12.789701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.899s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:35:12.790180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:35:12.792001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:35:12.792332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:12.792678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:12.793820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6995fa30-c298-43b0-9d5c-d1eb00cf56af]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:12.811611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62c93932-54ec-491e-a7a5-b93e47be8322]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:12.836427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddaf4af5-609d-4eec-bf88-f8d95cab415f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:12.867391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d13b8d7-29b7-46d3-a1f0-8331c3a0d356]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 141521, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:12.889033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7b010f5-f464-45cf-89ea-ccad61846c9d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:12.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2cfd82a4-6f39-42a6-b044-ed997dd2bffc]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:12.934402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a61961f-085e-4d92-9880-b4cf72048b5d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:12.960300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7883984-092a-4621-9e97-2a3b6e3c3974]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:12.961550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:12.962011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:12.962595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:35:12.962595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:35:12.984046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:35:12.984423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e83911ac-fbaf-439f-a540-f52bc6702198]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:12.988692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89b6a93d-dc34-40c0-a449-d129589f4658]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:12.989711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:35:13.022809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:35:13.023554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:35:13.025035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:35:13.033712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:35:13.035888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:35:13.100973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6882e307-af94-45a9-af3d-c324ab14b7b5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:13.133277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4eb7bc90-e601-4fc0-83b5-8252a7953940]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:35:13.133277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:35:13.133277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:35:13.133277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:35:13.133277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:35:13.133277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:35:13.133277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:35:13.133277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:35:13.133277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:35:13.133277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:35:13.133277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:35:13.133277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:35:13.133277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:13.135700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4eb7bc90-e601-4fc0-83b5-8252a7953940]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:13.137198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:13.156730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:35:13.160921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4739570f-270c-401a-a5b3-4ff8487fc937]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:13.160921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:35:13.160921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:35:13.160921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:35:13.160921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:35:13.161568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9621f3b1-71cf-4a3f-a702-930aa41169e2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:13.162558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c43c66b2-5a7f-4084-83fb-074d05714256]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:13.163985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[729db9f0-3beb-4927-99ed-dfc94983f35f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:13.193987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abd3c8b0-6fc4-4cf6-b040-449f196e9ba7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 300912, 'reachable_time': 28504, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 141543, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:13.199351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:35:13.199750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bc6420f-994a-469a-8542-faed163ff310]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:13.200687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:32:07Z", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:35:13.205682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id 7d6474b5-6182-401d-aa2e-d525cd468441. Wait time elapsed: 1.313 Jul 01 09:35:13.224409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 2, update_id 7d6474b5-6182-401d-aa2e-d525cd468441. Wait time elapsed: 1.314 Jul 01 09:35:13.224409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:35:15.081612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:15.082323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:15.082323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:15.082511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:16.033066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.831s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:35:16.033591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:35:16.035543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:35:16.036809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:16.037191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:16.039080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27f6a70f-ab8b-4a46-afe9-c7745efc2570]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.067255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ba5fef0-1eff-4fd2-9214-948a64fd6464]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.087663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[edbd601c-4b64-4cc0-a115-9d7adaff4513]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.133895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84dd7e3c-3f51-45a8-9347-73e7855c5085]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 141597, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.168507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e776e77-a0ff-4305-8eb3-29565dd588df]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.225534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfada82f-5a49-4507-b787-6a2daaeb9507]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.265747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06cf3d92-3e05-4efb-b79c-aa44d73e987a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.320639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b8c89ef-b234-44c5-9c71-74b840e22464]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.321368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:16.321788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:16.322451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:35:16.322921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(065872ae-5c22-41b4-9a11-b461d0aff77f) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:35:16.345893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:35:16.346533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[159fe53a-bb82-4f9e-b9be-a982fcdbd7f9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.350891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03695618-6e33-486f-859b-6e07dd6213a6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.352297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-065872ae-5c, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:35:16.392284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-065872ae-5c, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:35:16.393923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:35:16.394533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:35:16.395129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-065872ae-5c, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:35:16.395743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-065872ae-5c, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '065872ae-5c22-41b4-9a11-b461d0aff77f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ab:0e:2a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:35:16.449851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88d5e954-d8da-48f9-959d-7e2fcbc4fca6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[73544eda-5e1a-4b53-9571-376a54c7a94e]: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73544eda-5e1a-4b53-9571-376a54c7a94e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:35:16.510514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9376c0a3-79bc-448d-99b5-d9ca0049c366]: (4, ['qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:35:16.518757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:35:16.523525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:35:16.523525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6309d676-d46f-4b14-9cc9-4bdf2d4fcdaa]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.523525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59c392ed-16c5-489a-a884-75578ae98c01]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.523525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[193cbc2f-a859-4523-afaa-92fb980d919b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.523525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-66c3fb1b-6d91-420a-b893-ed61840c866f admin admin] Got routers updated notification :['22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:35:16.532081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5458b95e-c95d-4173-b078-6392f6a9fb51]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 301239, 'reachable_time': 30032, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 141618, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.533855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:35:16.534128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[155ab077-857b-4684-9f08-ae91b98baa4b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-065872ae-5c not found in namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3. Jul 01 09:35:16.535241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersTestManualDisk-381985282-router", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57" Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:00Z", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:07Z", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b09b9e0f-29ce-44ec-926d-d74f2c05ee7a", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:2c:9f:1d", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1367, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.57", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:01Z", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:14Z", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "065872ae-5c22-41b4-9a11-b461d0aff77f", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "92fd2355-76b9-4905-99de-c614594c0365", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ab:0e:2a", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1373, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:31:05Z", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:35:14Z", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "e7853c64e1714f88b6d625a103b68ea4", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "76888b0c-0ae0-4ead-977b-784ee4792d80", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:35:16.538082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 1, update_id d4665f5d-9f11-4e30-9f4d-7f8da51d359e. Wait time elapsed: 0.031 Jul 01 09:35:16.561234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 3, priority 1, update_id d4665f5d-9f11-4e30-9f4d-7f8da51d359e. Wait time elapsed: 0.031 Jul 01 09:35:16.561234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:35:17.109180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.572s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:35:17.109789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:35:17.111008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:35:17.111335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:17.111859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:17.113357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fcd7fb3-ee0d-4db8-9530-e9404bced40e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.135672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c320eac6-6cd8-4282-9509-09cf59615ead]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.167671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66d0e1b7-23ae-43a1-8b38-1efc02128336]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.209605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d249924-4efd-418a-b0f9-fae1159afc0f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 141622, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.233820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[769c8384-9625-49bd-88ba-287bd535237d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.265471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12c4f37a-b7d0-4de9-b958-9d689c97b3c4]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.295738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e85b4cd2-9c5c-43a6-aaeb-4225011f6e27]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.323762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22fbb461-81ec-4e58-b8fa-749395152d86]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.324488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:17.324860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:17.325305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:35:17.368841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ac0f9f1-508b-4a34-ae63-cfdabc99d7e7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 301347, 'reachable_time': 39692, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 141628, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.371283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:17.371905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:17.372675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:35:17.372802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:35:17.400591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa8c3cf2-909f-4309-80fd-90ba1a1c758e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 301347, 'reachable_time': 39692, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 141629, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.402727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:35:17.403717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:35:17.436925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05c15630-7d8c-46dc-8df3-b6f580a0ebf7]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.570093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-4b6a66e6-17ee-4ae8-89ef-2d1fcb0f97b7 admin admin] Got router deleted notification for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:35:17.631563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91967768-af64-430f-ad12-be6735b80255]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.664237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d057ed16-43bc-46b5-99c8-130ac4676a39]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.860213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8be9c27e-ce2a-4098-8a82-ad9b252761a1]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.860798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:35:17.861172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:35:17.861523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:35:17.861873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" released by "process_external" :: held 0.489s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:35:17.864440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:35:17.864774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:35:17.865603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:35:17.866139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:35:17.881454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e802a4e-d32d-46b1-b36d-c5c0ee94c793]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:35:17 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:35:17 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:35:17 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:35:17 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:35:17 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:35:17 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:35:17 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:35:17 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.911347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[925f6db9-de2d-4fc4-ab80-de448d3a6aa0]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.924387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49745232-08e2-4cef-bd5d-afb71e136288]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:35:17 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:35:17 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:35:17 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:35:17 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:35:17 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:35:17 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:35:17 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:35:17 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.962606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8231c28e-cc70-455a-ad6d-11d0db457553]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:17.962952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:35:17.963349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:35:17.963729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:35:17.964195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" released by "process_address_scope" :: held 0.099s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:35:17.964588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:35:17.965652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:35:17.965652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:35:17.967172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:35:17.967172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:35:17.984471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3ea9bea-188c-4cdb-883e-bd15c02873c7]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:35:17 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:35:17 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:35:17 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:35:17 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:35:17 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:35:17 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:35:17 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:35:17 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:18.050984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7c5735c-b141-42a7-b376-1108d93620ac]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:18.063731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7becfa96-d451-4e90-83a0-853a5a7898a8]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:35:18 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:35:18 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:35:18 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:35:18 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:35:18 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:35:18 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:35:18 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:35:18 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:18.126962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6bfb0428-c629-4761-abdb-116cb8350e73]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:18.127548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:35:18.128279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:35:18.128754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:35:18.129287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3" released by "apply_metadata_nat_rules" :: held 0.164s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:35:18.129939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.haproxy Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 Jul 01 09:35:18.130590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:35:18.133595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:35:18.179784 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3[141643]: Proxy listener started. Jul 01 09:35:18.184727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:35:18.185016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, update_id d4665f5d-9f11-4e30-9f4d-7f8da51d359e. Time elapsed: 1.649 Jul 01 09:35:18.185286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 1, priority 1, update_id e62c34a6-d123-4f54-8b6d-906455aa0905. Wait time elapsed: 0.614 Jul 01 09:35:18.185528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, action 1, priority 1, update_id e62c34a6-d123-4f54-8b6d-906455aa0905. Wait time elapsed: 0.615 Jul 01 09:35:18.185817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:35:18.186124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:35:18.226522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7e2bf3f-9652-4f2f-91aa-553f3c10a201]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:18.228341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffa65470-3726-40e9-9ccd-1c2d2c4be4c5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:18.229416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5891d0c0-0c23-4215-ae7c-d4fc29a69e41]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:18.229844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:35:18.230652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fabf3d4b-df5e-4137-8d47-16e858213845]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:18.276600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[711851ae-2fd0-404c-8586-10c912f1e6c4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 301347, 'reachable_time': 39692, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 141650, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:18.277830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:18.278281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:18.312290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7abadf4-2b4a-485b-8679-3e9bd46386fe]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 301347, 'reachable_time': 39692, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 141653, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:18.313253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:35:18.313739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:35:18.314053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:35:18.314302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:35:18.315039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:35:18.318193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00e8e881-3437-42bd-a91a-fadd507d751e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:18.357233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da72285d-f92c-49ca-a494-857e4336e56e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 301347, 'reachable_time': 39692, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 141656, 'error': None, 'target': 'qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:18.368621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:35:18.368905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bff56a16-7866-4ac2-a76e-9e4d30465695]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:18.369381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:35:18.369758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:35:18.370105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:18.370377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:18.370665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:18.371240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:18.371840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 22c747a2-9fe2-4e98-b4c8-b9ebe51a09e3, update_id e62c34a6-d123-4f54-8b6d-906455aa0905. Time elapsed: 0.186 Jul 01 09:35:28.420105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-4d6f5f37-8f78-48b8-9651-77e390149f83 tempest-RoutersTestDVR-1877934139 tempest-RoutersTestDVR-1877934139-project-admin] Got router deleted notification for 1c646a4a-1ca3-4db2-a65b-75f3918583c9 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:35:28.423098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 1c646a4a-1ca3-4db2-a65b-75f3918583c9, action 1, priority 1, update_id b2f1d503-2b3c-4bbc-806d-38ff23d0955c. Wait time elapsed: 0.001 Jul 01 09:35:28.427547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 1c646a4a-1ca3-4db2-a65b-75f3918583c9, action 1, priority 1, update_id b2f1d503-2b3c-4bbc-806d-38ff23d0955c. Wait time elapsed: 0.003 Jul 01 09:35:28.428673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 1c646a4a-1ca3-4db2-a65b-75f3918583c9 was not found. Performing router cleanup Jul 01 09:35:28.434936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a146d5a-abc7-4ae1-a7c9-4b083ca94cd6]: (4, ['qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:28.437965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:28.462820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:28.467601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 1c646a4a-1ca3-4db2-a65b-75f3918583c9, update_id b2f1d503-2b3c-4bbc-806d-38ff23d0955c. Time elapsed: 0.042 Jul 01 09:35:29.260996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-d71c0bab-ebdd-4fcb-b18a-513e0b1a45cb tempest-RoutersTestDVR-1877934139 tempest-RoutersTestDVR-1877934139-project-admin] Got router deleted notification for 6ecaf1e9-c555-45d5-a249-1409cd672b6e {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:35:29.262334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 6ecaf1e9-c555-45d5-a249-1409cd672b6e, action 1, priority 1, update_id 9c9ac3a1-ec7c-45ae-b5a1-6d154e73205f. Wait time elapsed: 0.001 Jul 01 09:35:29.262803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 6ecaf1e9-c555-45d5-a249-1409cd672b6e, action 1, priority 1, update_id 9c9ac3a1-ec7c-45ae-b5a1-6d154e73205f. Wait time elapsed: 0.001 Jul 01 09:35:29.263265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 6ecaf1e9-c555-45d5-a249-1409cd672b6e was not found. Performing router cleanup Jul 01 09:35:29.264892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af6fb63e-8902-4421-a4ea-0e33db2aea9f]: (4, ['qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:29.266173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:29.266661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:29.267119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 6ecaf1e9-c555-45d5-a249-1409cd672b6e, update_id 9c9ac3a1-ec7c-45ae-b5a1-6d154e73205f. Time elapsed: 0.005 Jul 01 09:35:30.895061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-0435a236-1a90-4db2-b56b-58f8b761ca47 tempest-RoutersTestDVR-1877934139 tempest-RoutersTestDVR-1877934139-project-admin] Got router deleted notification for 63631300-a17c-4af9-9a28-82254dd8a8f2 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:35:30.896717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 63631300-a17c-4af9-9a28-82254dd8a8f2, action 1, priority 1, update_id 9553dd39-d8bf-4f1d-b0dd-9aa6f1d06130. Wait time elapsed: 0.001 Jul 01 09:35:30.897432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 63631300-a17c-4af9-9a28-82254dd8a8f2, action 1, priority 1, update_id 9553dd39-d8bf-4f1d-b0dd-9aa6f1d06130. Wait time elapsed: 0.001 Jul 01 09:35:30.897874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 63631300-a17c-4af9-9a28-82254dd8a8f2 was not found. Performing router cleanup Jul 01 09:35:30.899194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c18fb3e3-924a-4945-8b4b-433069906d67]: (4, ['qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:30.900411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:30.900828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:30.901256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 63631300-a17c-4af9-9a28-82254dd8a8f2, update_id 9553dd39-d8bf-4f1d-b0dd-9aa6f1d06130. Time elapsed: 0.005 Jul 01 09:35:31.360047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-f7f151fd-37eb-4524-9ee4-29d551c2b1c9 tempest-RoutersTestDVR-1877934139 tempest-RoutersTestDVR-1877934139-project-admin] Got router deleted notification for 342f00ac-dd4b-4341-ab79-9a4c4c3c9b95 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:35:31.365612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 342f00ac-dd4b-4341-ab79-9a4c4c3c9b95, action 1, priority 1, update_id e1fba714-d54b-4fe3-bc7b-d0655794f954. Wait time elapsed: 0.001 Jul 01 09:35:31.365612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 342f00ac-dd4b-4341-ab79-9a4c4c3c9b95, action 1, priority 1, update_id e1fba714-d54b-4fe3-bc7b-d0655794f954. Wait time elapsed: 0.001 Jul 01 09:35:31.365612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 342f00ac-dd4b-4341-ab79-9a4c4c3c9b95 was not found. Performing router cleanup Jul 01 09:35:31.365612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5276c874-9bf5-4243-9bdc-e544b2580793]: (4, ['qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:35:31.365612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:35:31.365612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:35:31.365612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 342f00ac-dd4b-4341-ab79-9a4c4c3c9b95, update_id e1fba714-d54b-4fe3-bc7b-d0655794f954. Time elapsed: 0.004 Jul 01 09:35:35.117475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:35:59.591980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-9ad0c30c-5658-4373-897b-32ee2431692d tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['e1dc75ae-b350-4801-91dc-89215c41ca50'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:35:59.592934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 1, update_id b540b553-f4ee-4066-acb3-8686da5e4cc5. Wait time elapsed: 0.001 Jul 01 09:35:59.593185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 1, update_id b540b553-f4ee-4066-acb3-8686da5e4cc5. Wait time elapsed: 0.001 Jul 01 09:35:59.593502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:35:59.615680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-3a58eabe-5b87-4bb7-a1d3-8d6efd8b2b8a tempest-TestNetworkAdvancedServerOps-636282898 tempest-TestNetworkAdvancedServerOps-636282898-project] Got routers updated notification :['58317645-6e22-4a4b-8851-15323d1dba25'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:35:59.618163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 1, update_id e55002d5-e46d-468f-bde7-095cb6cd66dc. Wait time elapsed: 0.002 Jul 01 09:35:59.618665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 1, update_id e55002d5-e46d-468f-bde7-095cb6cd66dc. Wait time elapsed: 0.002 Jul 01 09:35:59.619161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:00.510783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.891s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:00.511274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:00.512378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:00.512705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:00.512984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:00.524001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93404ef8-8f7b-490f-be6b-6693cdf7a6e5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.536794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32436124-81ce-4c12-a239-7fa563486e01]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.559229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9ab0c43-62fc-4d7c-9e42-57de0b9041b0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.603666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8319a295-d89f-45c7-9a14-0d4c86d968c2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142292, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.628026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0fa3854-86b3-4565-afe1-eaa755033443]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.649785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1a51254-c4f8-4edf-ab86-3f49973304c7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.673148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74390605-8589-4fd0-8bfb-3e81c1932b42]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.701666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f959cd78-02b7-4901-9aca-42cf151fc0f0]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.704366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:00.704795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:00.705478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:00.705872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:00.727644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:00.728710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b19502fd-5d6b-4273-8aae-be40bc51b601]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.733295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8aabdeb3-42dd-4fb5-b684-5146e36a61b5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.734286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:00.774453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:00.775442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:00.776155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:00.777404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:00.778003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:00.840063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[468aef00-661b-480b-bca3-af3823243bb7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.871397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[247c417a-2881-4c1b-a335-d25f170e1813]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:00.871397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:00.871397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:00.871397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:00.871397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:00.871397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:00.871397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:00.871397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:00.871397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:00.871397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:00.871397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:00.871397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:00.871397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:00.872446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[247c417a-2881-4c1b-a335-d25f170e1813]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:00.872753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:00.874467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:00.877235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bad93929-b2be-43f4-8e63-e54f8dc6dc17]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.877235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:00.877436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:00.877605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:00.878126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:00.879156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da359283-1fe9-43d3-9592-87878092d30b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.880194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00701817-288f-4925-a8ed-ddcde8977934]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.881932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[217c90c3-e1ea-46d8-b05b-3b939b3c0991]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.909853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57bd65db-4a63-4f78-a46e-f2c793c23e32]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 305686, 'reachable_time': 18936, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142311, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.912469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:00.912829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d72e587c-f3c4-4f14-b18b-c23c7cab953c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:00.913914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:54Z", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:00.918104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id e55002d5-e46d-468f-bde7-095cb6cd66dc. Wait time elapsed: 1.302 Jul 01 09:36:00.941124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id e55002d5-e46d-468f-bde7-095cb6cd66dc. Wait time elapsed: 1.302 Jul 01 09:36:00.941124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:02.261546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.668s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:02.262118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:02.263260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:02.264016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:02.264432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:02.265622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0e02bed-db1d-4096-ae3c-8994b9f79eac]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.278454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0358fb83-de29-446b-b21f-e93f33b56c7d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.299241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a8b120f-a3d2-40ca-b381-9cc9e050423e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.327610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c52e3fdb-5763-45ed-b776-ad2a031fb4da]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142314, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.348562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[490f1104-06e6-4605-9cfd-6d67efc4cb35]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.368842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e17c973a-7132-4796-998d-2211df560865]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.389202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de1f582c-4da1-4668-93c6-98b8a2bbcdb9]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.410418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47c83ac0-27b9-436a-ab81-027299eaf36e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.410991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:02.411266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:02.411595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:02.411869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:02.428357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:02.429067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66e3ad86-9d58-4fd6-944e-4f7036e588af]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.443742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2685edbe-5f79-431e-8383-056d7b4cd166]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.445563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:02.479156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:02.479869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:02.481008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:02.481576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:02.481967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:02.540462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3df9e008-e1a9-49f9-8fd9-0b13b54b3187]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.581328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2a677652-0ec0-44eb-8f66-063a8d07ec48]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:02.581328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:02.581328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:02.581328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:02.581328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:02.581328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:02.581328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:02.581328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:02.581328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:02.581328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:02.581328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:02.581328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:02.581328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:02.582562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a677652-0ec0-44eb-8f66-063a8d07ec48]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:02.583839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:02.586200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:02.589519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a33efaca-224b-4a75-8701-52f3bd3ff308]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.590413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:02.590956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:02.591519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:02.592095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:02.593442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f0c001f-b0cc-43b8-a16d-932e27d3ac72]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.594557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a5d5238-efb0-4213-9abd-2b466c95e79c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.595743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1784067f-6c67-4c0d-8ce0-5df934bd2661]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.623638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64b8bb7f-fd34-4516-8616-b6c70dfbf18d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 305859, 'reachable_time': 15812, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142333, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.630217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:02.630619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af5e20ee-82e9-44d9-ab41-1e76dffbf22f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:02.631822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:47Z", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:02.635107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:02.649340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b540b553-f4ee-4066-acb3-8686da5e4cc5. Wait time elapsed: 3.043 Jul 01 09:36:02.649340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b540b553-f4ee-4066-acb3-8686da5e4cc5. Wait time elapsed: 3.044 Jul 01 09:36:02.649340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:03.381580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.744s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:03.382177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:03.382807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:03.383139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:03.383451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:03.385544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7278e1e8-36cd-48ec-ac87-46131298d846]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.405571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[571d9db2-abf0-49ae-a17c-5efed12c4b89]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.427860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffd447ce-b069-4070-939b-215a96def9a5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.458135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0dbb303a-7f8b-4f4e-877e-04e78729a418]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142336, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.479056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41706d6d-459f-4a5c-9bd3-7d05d2556ebf]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.501882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72b8b110-24bb-4180-a5fa-77ba34de247c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.529893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8b39bad-3b21-4f46-9653-7a5d324f5337]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.553038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a3fd1ea-cccb-4ad5-9cea-0094fd73c5da]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.554176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:03.554598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:03.555398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:03.555518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2f599072-31a6-4456-ac1e-cd5f64afdea5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:03.574462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:03.574901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74a4a8e6-d180-493b-8eb8-59f8e62e0849]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.582396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13cb6251-44bc-4ff6-af66-a343de8455d9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.584143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2f599072-31, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:03.621051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2f599072-31, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:03.624929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:03.625407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:03.625872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2f599072-31, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:03.626865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2f599072-31, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2f599072-31a6-4456-ac1e-cd5f64afdea5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:80:8e:a3'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:03.681364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c275c51f-5b0d-4c8c-a6e4-de6b550eb84e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6b84e1e5-1be8-4648-b8c0-7b6c0343f5d1]: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:03.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:03.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:03.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:03.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:03.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:03.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:03.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:03.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:03.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:03.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:03.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:03.716277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:03.717308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b84e1e5-1be8-4648-b8c0-7b6c0343f5d1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:03.717467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:03.719470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:03.723342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[632a9d5d-ac23-41d9-b25c-8246428f4aeb]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.723342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:03.723342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:03.723342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:03.723342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:03.724378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a145e578-89c7-4747-b785-5465f533e34a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.725693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c1c013b-17bf-4f7d-826e-9c6ec7ae9020]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.728396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f65523f9-2d78-4367-9132-6cd3984389ca]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.753649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2841a663-3b1b-4d88-9d93-492203b6ea4a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 305972, 'reachable_time': 28800, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142355, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.762736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:03.763026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24df45f3-e52d-4315-8586-2934c2e832b9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2f599072-31 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:03.764102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2f599072-31a6-4456-ac1e-cd5f64afdea5", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "46c1954f-b6ea-4704-8bef-168b5666d084", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:80:8e:a3", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1295, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:15Z", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:47Z", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "718a33be-a4b3-43e7-879a-f046f5f566cb", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:03.767841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b540b553-f4ee-4066-acb3-8686da5e4cc5. Wait time elapsed: 4.173 Jul 01 09:36:03.789133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 2, update_id b540b553-f4ee-4066-acb3-8686da5e4cc5. Wait time elapsed: 4.173 Jul 01 09:36:03.789133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:04.576728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.657s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:04.577728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:04.579612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:04.580682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:04.581690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:04.584573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92557714-645e-453f-91c4-82355913db67]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:04.615631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c45b52ec-cff4-4099-939b-dc479529b3d7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:04.647158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[298a68ad-ec78-44f8-ab41-924422c42f89]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:04.700655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6de794e6-95a9-4f35-8007-99bfdfd67941]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142360, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:04.727783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22a31b13-8b3e-4ad0-97ef-025f68ef0925]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:04.756549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbccd429-bb6c-475d-845a-0fb5011967d0]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:04.787517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f649e1f-abda-4b77-9eb9-4dba993aebef]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:04.818462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c13d3276-a323-47fc-9170-cf1ae6975dd1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:04.819752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:04.820504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:04.822032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:04.822501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(23815fb1-ba72-4865-909e-247dd181dfe5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:04.826370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-9b9c8750-4775-400c-82c9-1b9ddb8695d7 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['e1dc75ae-b350-4801-91dc-89215c41ca50'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:36:04.846464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:04.846940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[184a64af-4430-4688-b7df-8ca0281875e3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:04.851887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b3fd387-d2cc-4566-b628-cc2fc2d3ddff]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:04.852980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-23815fb1-ba, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:04.888697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-23815fb1-ba, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:04.889747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:04.890154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:04.890552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-23815fb1-ba, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:04.890939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-23815fb1-ba, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '23815fb1-ba72-4865-909e-247dd181dfe5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:d2:8a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:04.952158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43a10620-d920-4d16-9a8a-0f318dde8a8e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:04.992966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[88f15df1-d4eb-49bf-89b0-1960ec4eb6f0]: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:04.992966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:04.992966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:04.992966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:04.992966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:04.992966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:04.992966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:04.992966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:04.992966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:04.992966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:04.992966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:04.992966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:04.992966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:04.994438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88f15df1-d4eb-49bf-89b0-1960ec4eb6f0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:04.996026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:04.998841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:05.002676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc09190d-e791-4cbd-8a18-7e299c6a8a18]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:05.004084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:05.004505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:05.004855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:05.005292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:05.006495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa754b03-6718-4e76-b680-bb3f89665840]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:05.007635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e25341b-4c55-48bc-8e62-35320af045e5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:05.008777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[250af6a2-510b-4027-8edd-bf9ac3ccdf55]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:05.028212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91585d54-5190-4444-97ff-3b96e54bf74b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 306096, 'reachable_time': 33672, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142380, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:05.034465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:05.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d1a63ce-762e-4d83-a035-a715002d1fc6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-23815fb1-ba not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:05.036023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:16Z", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "23815fb1-ba72-4865-909e-247dd181dfe5", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:d2:8a", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1289, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:14Z", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:31:54Z", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51f0b40c-c390-4290-b98e-9ccf3ac4dd38", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:05.039669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:05.062084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id e55002d5-e46d-468f-bde7-095cb6cd66dc. Wait time elapsed: 5.431 Jul 01 09:36:05.062084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 2, update_id e55002d5-e46d-468f-bde7-095cb6cd66dc. Wait time elapsed: 5.432 Jul 01 09:36:05.062084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:06.330841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.283s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:06.331489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:06.332167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:06.332605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:06.333121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:06.334421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a882cdb-d8f6-4c18-b239-d80e583a4b3b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.353023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[279c3c24-522c-4963-9681-7bcf70641e01]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.374556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4bb4b52-21a0-4027-a7a2-f5dc5ca9353b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.411139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5db980d2-fee1-4140-9e8c-e8e6a81edc2b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142481, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.437389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e0c5082-5388-40ef-900e-906e88f98319]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.458798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9c69653-69a3-4b88-a5e3-4d9a4bc4b6ad]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.480123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d17dbf14-a1b9-4f72-b899-55796281208e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.502939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[704f2b3b-b322-482e-b6d4-a351d6d0a512]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.503555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:06.503831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:06.504094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:06.524240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4adaba65-4e0e-4be7-8723-dcc2c5024789]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 306266, 'reachable_time': 42956, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142487, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.525163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:06.525506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:06.526005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:06.527045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:36:06.527410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:06.527830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:06.528112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '6abcecf7-8aba-4321-83e2-43a0ac8d9c3a', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:48:1c:04', 'admin_state_up': True, 'status': 'ACTIVE', 'device_id': '58317645-6e22-4a4b-8851-15323d1dba25', 'device_owner': 'network:router_gateway', 'standard_attr_id': 1283, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.248', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:30:07Z', 'updated_at': '2022-07-01T09:30:17Z', 'revision_number': 4, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-6abcecf7-8a), ns(qrouter-58317645-6e22-4a4b-8851-15323d1dba25) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:36:06.545449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:06.545965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55cb3ac6-6172-412c-bbe9-0babd621fdcd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.549432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bd43cb6-a0a1-4408-83b6-366944bd724c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.550577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-6abcecf7-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:06.579408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-6abcecf7-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:06.581562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-6abcecf7-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:06.582184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-6abcecf7-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:06.582777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-6abcecf7-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:06.583383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-6abcecf7-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6abcecf7-8aba-4321-83e2-43a0ac8d9c3a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:48:1c:04'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:06.615018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8ba01b9-490b-4296-a848-8490263df2c2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.630543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c4f9d0a9-6dfd-43b0-bdab-56d8d9c0dad2 tempest-TestNetworkAdvancedServerOps-636282898 tempest-TestNetworkAdvancedServerOps-636282898-project] Got routers updated notification :['58317645-6e22-4a4b-8851-15323d1dba25'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:36:06.658750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[501ec559-35a6-4016-85ba-3ff9485790f4]: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:06.658750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:06.658750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:06.658750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:06.658750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:06.658750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:06.658750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:06.658750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:06.658750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:06.658750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:06.658750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:06.658750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:06.658750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:06.660624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[501ec559-35a6-4016-85ba-3ff9485790f4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.661433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:06.662204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:36:06.681686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d19f0eae-8b9d-40fc-a637-e8c0e0deaeb0]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.874328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8943dc92-8831-4c22-8fac-11533f48cae5]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:06.886704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea5a96a5-5b20-477b-a187-1901640845de]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.062663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58f6e048-640c-40e9-a4a5-ba8ab172827f]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.064316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:36:07.065151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:07.065541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:07.066075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" released by "process_external" :: held 0.539s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:07.067393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:07.069686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:07.072393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50038913-c435-45b8-aff3-0b0f41892788]: (4, ['qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.072393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:07.072393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:07.072393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:07.072393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:07.075061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0abbb049-b719-4d47-8651-b4ddc4659960]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.076439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3cdd356e-fc20-419b-986e-ca144c501b7d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.077532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf7e97a6-68c8-4437-9ca6-c660e9ed1024]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.099984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75fa7de8-2aa6-4b29-9e08-71ba6921bd9c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 306266, 'reachable_time': 42956, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142504, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.106669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:07.107248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66a478c0-6aed-4880-a230-7aff26076df4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 58317645-6e22-4a4b-8851-15323d1dba25: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6abcecf7-8a not found in namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25. Jul 01 09:36:07.108896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1049722070", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:06Z", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:04Z", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "8619fd3f7f684024b7936c10fa693612", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6abcecf7-8aba-4321-83e2-43a0ac8d9c3a", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:1c:04", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "58317645-6e22-4a4b-8851-15323d1dba25", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1283, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:17Z", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 1, update_id d0271b6c-eeab-4e27-9c7b-104e7a547ea8. Wait time elapsed: 0.478 Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 3, priority 1, update_id d0271b6c-eeab-4e27-9c7b-104e7a547ea8. Wait time elapsed: 0.479 Jul 01 09:36:07.113092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:07.482128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.716s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:07.483156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e1dc75ae-b350-4801-91dc-89215c41ca50 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:07.484259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e1dc75ae-b350-4801-91dc-89215c41ca50), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:07.484591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:07.484878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:07.487233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c6e12ed-9198-4354-9d86-f5fa9c535c46]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.502526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d290243-6e46-4080-879f-e5343a88cebd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.527172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4be883a5-7ba5-40df-a87c-3b3e26d72aae]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.560522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac723f61-aeca-4664-8df6-fcae0aa656d2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142507, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.584641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc92cc8e-80cb-44f5-bb6d-d4b249a6c8a1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.610059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d202b6cc-90fc-4959-b6c5-aeb798130599]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.635540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09958a43-7ddf-4171-836e-aeb23a139d54]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.661456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-911972df-78b7-4792-9b9f-6902a4f16076 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got router deleted notification for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:36:07.662275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[642e5eeb-baf4-43fb-93ff-9eed8a384afc]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.663236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:07.663543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:07.664173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:07.688962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d15af4e-735a-4741-abea-91c4c66983d6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 306382, 'reachable_time': 31888, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142513, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.691757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:07.693311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:07.693921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:07.694727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:36:07.696455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:07.696871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:07.697389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '105dbb76-6459-4ce9-a126-c111de1babfc', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:9e:72:fc', 'admin_state_up': True, 'status': 'ACTIVE', 'device_id': 'e1dc75ae-b350-4801-91dc-89215c41ca50', 'device_owner': 'network:router_gateway', 'standard_attr_id': 1284, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.160', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:30:08Z', 'updated_at': '2022-07-01T09:30:18Z', 'revision_number': 4, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-105dbb76-64), ns(qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:36:07.718303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:07.718695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f22d8b8-dc86-4004-8f62-ac64bd496381]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.722689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c9227f9-5fff-4964-97d3-7c8127bc8a8f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.723697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-105dbb76-64, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:07.758340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-105dbb76-64, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:07.758848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-105dbb76-64, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:07.759314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-105dbb76-64, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:07.760653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-105dbb76-64, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:07.761416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-105dbb76-64, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '105dbb76-6459-4ce9-a126-c111de1babfc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:72:fc'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:07.807160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d183f33-c1fe-42bb-b160-6b6eff03c6df]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.835378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0f29045f-1273-47e7-99c5-8995954d4380]: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:07.835378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:07.835378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:07.835378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:07.835378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:07.835378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:07.835378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:07.835378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:07.835378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:07.835378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:07.835378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:07.835378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:07.835378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:07.836532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f29045f-1273-47e7-99c5-8995954d4380]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:07.837297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:07.839711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:36:07.857181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cef08122-7a13-4452-acda-ac9bd3be0006]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.065766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffddd52e-c041-474a-8f95-c05efe528408]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.093132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29fc4a35-99da-4134-b718-3b0dbc37ad77]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.326734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be9311c9-806a-44c3-a759-4bd22c563c18]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.327556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:36:08.327837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:08.328770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:08.329061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50" released by "process_external" :: held 0.635s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:08.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:08.332147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:08.334678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9671bfe7-ffca-4d79-bfc1-001128cfc4df]: (4, ['qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.334802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:08.335713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:08.335713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e1dc75ae-b350-4801-91dc-89215c41ca50 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:08.336207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e1dc75ae-b350-4801-91dc-89215c41ca50.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:08.337796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a2061c5-e607-4cfd-a320-a5021e52f7e5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.338763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e900ccc-e632-4fe2-bc6a-935ba4e9ef66]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.340239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f2da24b-625f-4ed9-a748-f4bac6295805]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.365518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e8a7c98-9150-4412-a3d1-b27a9344ceb2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 306382, 'reachable_time': 31888, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142532, 'error': None, 'target': 'qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.368189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:08.368517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d3c45ee-a223-4d1f-a71d-5a72c5f6831f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e1dc75ae-b350-4801-91dc-89215c41ca50: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-105dbb76-64 not found in namespace qrouter-e1dc75ae-b350-4801-91dc-89215c41ca50. Jul 01 09:36:08.369648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-453340139", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160" Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:07Z", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:04Z", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "105dbb76-6459-4ce9-a126-c111de1babfc", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:72:fc", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e1dc75ae-b350-4801-91dc-89215c41ca50", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1284, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.160", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:30:08Z", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:30:18Z", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 1, update_id 9e02e97d-d048-410d-9fef-52d14d0bead0. Wait time elapsed: 3.544 Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e1dc75ae-b350-4801-91dc-89215c41ca50, action 3, priority 1, update_id 9e02e97d-d048-410d-9fef-52d14d0bead0. Wait time elapsed: 3.544 Jul 01 09:36:08.373856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:08.421833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.051s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:08.422242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router e1dc75ae-b350-4801-91dc-89215c41ca50 was not found. Performing router cleanup Jul 01 09:36:08.423983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f21d013-605a-44a0-8715-cd17dda8bb96]: (4, ['qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.425293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:08.425673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:08.426111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for e1dc75ae-b350-4801-91dc-89215c41ca50, update_id 9e02e97d-d048-410d-9fef-52d14d0bead0. Time elapsed: 0.055 Jul 01 09:36:08.669399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-fb3334f1-8c53-439c-8cea-991ff44f8be4 tempest-TestNetworkAdvancedServerOps-636282898 tempest-TestNetworkAdvancedServerOps-636282898-project] Got router deleted notification for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:36:08.761571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.651s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:08.761854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 58317645-6e22-4a4b-8851-15323d1dba25 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:08.763300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:08.763569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:08.763917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:08.765132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2415e4b5-442a-4c62-8fbb-37c533bbf205]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.781533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02ea2f53-1008-4a16-8eb8-02e0eabccdbb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.808956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[161da4c1-11e2-4b16-a0c5-6ea87f77a033]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.845920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee2eb4c2-9728-451d-9643-308cbd21b7d1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142535, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.873143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17ae7374-60c7-4564-8a0b-2af3dde41a99]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.895702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[459d4980-4375-49d0-a6ad-9facb77b54b2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.921371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7d54faf-4b24-4fc5-a3ee-69ac29d9a0f5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.947161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58db04a8-7ada-4dcb-aff7-1a7ab08d3983]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.948054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:08.949043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:08.949539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:08.980099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c3059fc-345d-407d-af47-c97457efe924]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 306510, 'reachable_time': 38460, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142541, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:08.981672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:08.982164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:08.983274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:08.983812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:36:09.011099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2aca47c-0cd8-4bf4-9d57-3ebd0a521cda]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 306510, 'reachable_time': 38460, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142542, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:09.012068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:09.012604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:36:09.024233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ab1363c-ad1a-45a0-9d9b-42b435ddd9eb]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:09.198361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5c39ff0-fcf1-4357-93c3-465170e57750]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:09.210514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c26e864-45d8-4fec-8c1e-4b14956f5e37]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:09.445401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f64d4cd-f429-48a5-beba-8f8541155cad]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:09.446783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:36:09.447320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:09.448129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:09.448566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" released by "process_external" :: held 0.465s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:36:09.449263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:09.449671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" acquired by "process_address_scope" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:36:09.450822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:09.451581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:36:09.481566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e407966-4455-4072-a63d-649f0ff438cb]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:09.552035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bff9c1a3-f538-40b8-b8a2-4abc1c479d08]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:09.596774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f55ea42-4d4a-4726-b89e-c8ef3489b5d2]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:09.666387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ec671cf-3403-4450-9b30-d2b05f3c8fd1]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:09.667041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:36:09.667885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:09.668148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:09.668438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" released by "process_address_scope" :: held 0.219s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:36:09.668807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (58317645-6e22-4a4b-8851-15323d1dba25), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:09.669271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:09.669521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:36:09.670820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:09.671542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:36:09.704816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[043062d0-a017-44cd-a361-28e036e310b5]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:09.786812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97ae2ed2-fc47-4e81-9353-919b9cbd092e]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:09.813112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20320b46-0b4d-4c36-88ab-aaa9c833cf84]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:36:09 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:36:09 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:09.893674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9b055ae-d1a4-48dc-801d-ef9fa85ddbb1]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:09.895312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:36:09.895963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:09.896734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:09.897362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-58317645-6e22-4a4b-8851-15323d1dba25" released by "apply_metadata_nat_rules" :: held 0.228s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:36:09.898255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-58317645-6e22-4a4b-8851-15323d1dba25 Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.haproxy Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 58317645-6e22-4a4b-8851-15323d1dba25 Jul 01 09:36:09.899096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:36:09.903804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/58317645-6e22-4a4b-8851-15323d1dba25.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:36:09.943955 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-58317645-6e22-4a4b-8851-15323d1dba25[142558]: Proxy listener started. Jul 01 09:36:09.949061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 58317645-6e22-4a4b-8851-15323d1dba25 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:36:09.950657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 58317645-6e22-4a4b-8851-15323d1dba25, update_id d0271b6c-eeab-4e27-9c7b-104e7a547ea8. Time elapsed: 2.840 Jul 01 09:36:09.951300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 58317645-6e22-4a4b-8851-15323d1dba25, action 1, priority 1, update_id 251f87c1-b1e2-4efe-b5c7-228803cd8b62. Wait time elapsed: 1.281 Jul 01 09:36:09.951570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 58317645-6e22-4a4b-8851-15323d1dba25, action 1, priority 1, update_id 251f87c1-b1e2-4efe-b5c7-228803cd8b62. Wait time elapsed: 1.281 Jul 01 09:36:09.951881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 58317645-6e22-4a4b-8851-15323d1dba25 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:36:09.952201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (58317645-6e22-4a4b-8851-15323d1dba25), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:10.015201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6b97f3c-b98d-47e5-8aee-cf93da8d04c7]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:10.017955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a7281af-e901-4f20-b540-1b36f356b1fe]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:10.019462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ab2ab10-953e-4e2e-8df1-2cd484ab863a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:10.020065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:36:10.021094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[578bd114-df95-4d8a-860c-8e97a234c3ca]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:10.049241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab51010d-de85-49b1-a9b2-03f2cbd000f4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 306510, 'reachable_time': 38460, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142564, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:10.051097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:10.051410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:10.078163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0bf6caf2-1d4b-4974-9c56-114e114624f8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 306510, 'reachable_time': 38460, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142565, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:10.082068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:36:10.083309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:10.083825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/58317645-6e22-4a4b-8851-15323d1dba25.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:10.084262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:10.085260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 58317645-6e22-4a4b-8851-15323d1dba25 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:36:10.086739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f001404a-002b-4595-97e7-799040c8ffe7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:10.112124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d3354c5-7288-4134-89cd-891cb03e1000]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 306510, 'reachable_time': 38460, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142566, 'error': None, 'target': 'qrouter-58317645-6e22-4a4b-8851-15323d1dba25', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:10.119511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-58317645-6e22-4a4b-8851-15323d1dba25 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:10.119772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6df2c09c-9500-46f1-97bb-743bfbc014ab]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:10.120199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 58317645-6e22-4a4b-8851-15323d1dba25 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:36:10.120528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (58317645-6e22-4a4b-8851-15323d1dba25), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:10.121529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:10.123258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:10.123920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:10.124496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:10.124947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 58317645-6e22-4a4b-8851-15323d1dba25, update_id 251f87c1-b1e2-4efe-b5c7-228803cd8b62. Time elapsed: 0.174 Jul 01 09:36:10.987794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ab1b7a48-6b9b-4860-92b4-7270f741df6d tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['203e4741-69b9-468f-97b4-76a7d63b11b8'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:36:10.988951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 1, update_id 431852e6-8d33-429b-a28c-5c863668fe44. Wait time elapsed: 0.001 Jul 01 09:36:10.989292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 1, update_id 431852e6-8d33-429b-a28c-5c863668fe44. Wait time elapsed: 0.001 Jul 01 09:36:10.990143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:13.888106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.898s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:13.889014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:13.890348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:13.890830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:13.891264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:13.893314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efb53cb3-dddb-4dc8-b0ba-c13c02e4e274]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:13.909454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf0346aa-5b9f-4cdd-9122-0a38b7473239]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:13.934373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32a9b2d9-48de-484c-9907-e66f9ee4c5b9]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:13.978275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b916f7dd-971b-4146-8396-51fa4ee09fcb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142599, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.021325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[773fe0af-3231-4a46-b966-783c0bce33fb]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.053993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1d82764-f04e-42e7-bfcd-17d84a226661]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.103706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3518609e-40a0-445d-a672-afb027b68edf]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.146531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78e93939-366c-42b7-8572-f00275c10257]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.147128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:14.147549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:14.147828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:14.203544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d52451af-c362-429b-ae93-8c765358b841]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 307024, 'reachable_time': 42076, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142612, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.212915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:14.212915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:14.212915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:14.212915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:36:14.212915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:14.212915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:14.212915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '95238a63-f36c-4bb6-8f99-a580b7ab2a66', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:6e:e1:05', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '203e4741-69b9-468f-97b4-76a7d63b11b8', 'device_owner': 'network:router_gateway', 'standard_attr_id': 1996, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.184', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:36:09Z', 'updated_at': '2022-07-01T09:36:12Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-95238a63-f3), ns(qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:36:14.279048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-95238a63-f3 not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:14.279347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf0267a4-4816-48b8-941f-9ee4915deead]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.282120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8fc446c-28b1-4cfc-9139-19094439efb9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.283087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-95238a63-f3, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:14.283488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:36:14.284237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-95238a63-f3, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:14.286534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-95238a63-f3, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:14.286534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-95238a63-f3, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:14.286534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-95238a63-f3, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:14.286534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-95238a63-f3, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '95238a63-f36c-4bb6-8f99-a580b7ab2a66', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6e:e1:05'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:14.328747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1cd9b0f-d15f-4337-85ea-c73c45523d3c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.378771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c2bb8702-1c4b-4960-afdc-d32bac8e4136]: Network interface qg-95238a63-f3 not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:14.378771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:14.378771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:14.378771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:14.378771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:14.378771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:14.378771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:14.378771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:14.378771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:14.378771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:14.378771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:14.378771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:14.378771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-95238a63-f3 not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:14.380191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2bb8702-1c4b-4960-afdc-d32bac8e4136]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-95238a63-f3 not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.380884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:14.381546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:36:14.420884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b63c17ab-ba77-4593-98d7-80c9b922b118]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.569668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-3047ce13-a89d-4fb3-90d5-83aee8737fad tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['203e4741-69b9-468f-97b4-76a7d63b11b8'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:36:14.639063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d91786e-947d-4ceb-9329-ab7e4ab10468]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.671247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac723f47-a454-402c-a50f-8ef0fda61cdf]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.697838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-f5da0177-2b60-49a1-8705-f56e0720ee98 admin admin] Got routers updated notification :['9e1a30da-e2f9-4cd7-a55b-cc3a6e174576'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:36:14.699423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 1, update_id 01635dd2-a624-4888-bb4b-9c19d82d5c33. Wait time elapsed: 0.001 Jul 01 09:36:14.699657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 1, update_id 01635dd2-a624-4888-bb4b-9c19d82d5c33. Wait time elapsed: 0.002 Jul 01 09:36:14.700153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:14.879586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ffb6b33-83d5-44ae-89c9-47783644f848]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.880595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:36:14.881136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:14.881795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:14.882379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" released by "process_external" :: held 0.672s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-95238a63-f3 not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-95238a63-f3 not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-95238a63-f3 not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:14.883988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-95238a63-f3 not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-95238a63-f3 not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:14.888859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:14.897642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f14207af-9636-431d-a05a-081e9a40d26d]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.898585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:14.899206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:14.899628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:14.900111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:14.901391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58b4f1a1-3222-4087-9094-930d2e1a0171]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.903777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6b3b82a-7321-4b42-8f3a-626bbc56ade0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.905978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[980f221d-1f06-46da-ac84-b3ad940a4c31]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.941383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbdbb795-38a5-4c36-9104-89628db73ace]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 307024, 'reachable_time': 42076, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142654, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.949901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:14.950193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[743e6cc9-ea3b-4bbb-b3cd-5b57190d7ec6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-95238a63-f3 not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-95238a63-f3 not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:14.951299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:10Z", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:12Z", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 1, update_id c9266c6b-246e-4b1a-b825-2e8237beecb6. Wait time elapsed: 0.382 Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 1, update_id c9266c6b-246e-4b1a-b825-2e8237beecb6. Wait time elapsed: 0.383 Jul 01 09:36:14.956808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:15.082368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:15.082624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:15.082931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:15.083171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:15.119624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:36:16.289620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.589s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:16.290594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:16.291560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:16.291895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:16.292359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:16.293588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b8655ad-42ab-409f-be96-591b613ca2c2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.318462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[361d5656-acad-4402-a8a7-406d47275aa9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.360077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa66dbed-6ba3-49dc-9a24-5026759be1a9]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.411089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a7400e9-f1b1-40f9-b725-e76634616dec]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142661, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.450751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b74f12f-9ffd-4215-aa58-96cbb36566c8]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.488174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ed195bf-d369-495a-8271-bf0683a80123]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.535911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2031303-ab77-4cf1-8d7e-c58e556d5222]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.578248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c876b5d-75a5-467d-9663-eae8efecd546]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.580297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:16.580892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:16.581409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:16.607898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c613421-032f-4a81-9280-7a4ff16c59f4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 307267, 'reachable_time': 42588, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142667, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.609428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:16.609953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:16.611204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:16.612345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:36:16.612893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:16.613481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:16.614368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '782e9fbf-3821-41cc-853b-e44a675ba6ac', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:53:28:e1', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'device_owner': 'network:router_gateway', 'standard_attr_id': 2015, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.76', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:36:13Z', 'updated_at': '2022-07-01T09:36:15Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-782e9fbf-38), ns(qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:36:16.637278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-782e9fbf-38 not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:16.638338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd19908e-7ae1-46f6-a8a3-bedb377e4b51]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.646430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2adf0db2-b47a-46b9-86cd-3f578bd9cf6f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.648477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-782e9fbf-38, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:16.649306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:36:16.650853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-782e9fbf-38, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:16.651889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-782e9fbf-38, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:16.653326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-782e9fbf-38, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:16.654408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-782e9fbf-38, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:16.655081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-782e9fbf-38, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '782e9fbf-3821-41cc-853b-e44a675ba6ac', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:53:28:e1'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:16.711935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b680703a-dc82-4a4d-aa33-41a76fcf4763]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.757825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c45352ed-8a25-44d2-a892-08b67ed06b59]: Network interface qg-782e9fbf-38 not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:16.757825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:16.757825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:16.757825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:16.757825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:16.757825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:16.757825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:16.757825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:16.757825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:16.757825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:16.757825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:16.757825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:16.757825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-782e9fbf-38 not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:16.758777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c45352ed-8a25-44d2-a892-08b67ed06b59]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-782e9fbf-38 not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.758777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:16.759408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:36:16.798915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bfa4c16-0d3b-4232-b4cb-4b6952ccd570]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.970920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31a53bd5-2312-4e40-bb9a-4fb53bea6582]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:16.994952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f36b42e8-b8fe-46c0-830c-1d3bd3ccd269]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:17.072376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-a0437d6f-ec70-4cbc-9209-dcac4d820e76 admin admin] Got routers updated notification :['9e1a30da-e2f9-4cd7-a55b-cc3a6e174576'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:36:17.192119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff7c2a4c-d4c4-4dba-a599-a2356c9f9a1b]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:17.192698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:36:17.193002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:17.193317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:17.193588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576" released by "process_external" :: held 0.582s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-782e9fbf-38 not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-782e9fbf-38 not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-782e9fbf-38 not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:17.194377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-782e9fbf-38 not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-782e9fbf-38 not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:17.197308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:17.199989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50b71940-2538-4430-bfe5-5dc9e0ae96a4]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:17.199989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:17.199989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:17.199989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:17.199989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:17.206545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10c49ef1-d955-4519-807b-4e7767fdedfe]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:17.206545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bc5b1de-6d50-42c1-8267-665947b1b2d9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:17.206545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e46490c3-72eb-42be-b8cf-e6789d26934d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:17.230999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7458d1d-3e19-4c72-9819-874a8053a65e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 307267, 'reachable_time': 42588, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142684, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:17.238608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:17.238873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d08b703-d14a-4139-b9b9-040e8706e5be]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-782e9fbf-38 not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-782e9fbf-38 not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:17.240245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:15Z", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 1, update_id c9419d93-e4b7-4bc6-b96e-6b06ff675fa5. Wait time elapsed: 0.174 Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 1, update_id c9419d93-e4b7-4bc6-b96e-6b06ff675fa5. Wait time elapsed: 0.174 Jul 01 09:36:17.243471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:18.974698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.733s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:18.975308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:18.977236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:18.977579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:18.977865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:18.979286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd817cd3-1ab4-45a6-ad91-beebfdd843c6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:18.996312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2246be94-04ca-481c-a295-878604cf9c94]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.020514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1164eef9-6a9a-47f1-9475-8a1e0b259800]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.055945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a14d0f91-ef36-4285-a413-d177f125051b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142689, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.078449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b20f1c17-119c-4876-ab4c-a14bd9667f6b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.100744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18c21084-542e-4a09-b163-e554d87b9d7b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.122867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d5fe878-45fd-40fe-a472-9cffb15245e6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.135602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddee2f16-5c05-4554-b6e2-033bd9c60943]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.136665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:19.137255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:19.137816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:19.138163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:19.153763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:19.154129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06a3884e-3782-4de7-9014-f6ec8a38463b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.157014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7dec4ca4-712c-4c2a-a2a9-fccd4c22ea13]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.158912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:19.159322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:36:19.160246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:19.160973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:19.161425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:19.162177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:19.162623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:19.217569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70debc19-2622-489b-96b5-e0799da45179]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.258208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6adb03dc-0bf9-4eed-a9f9-ab09efdb2850]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:19.258208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:19.258208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:19.258208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:19.258208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:19.258208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:19.258208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:19.258208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:19.258208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:19.258208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:19.258208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:19.258208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:19.258208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:19.260393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6adb03dc-0bf9-4eed-a9f9-ab09efdb2850]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:19.261495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:19.263123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:19.265354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1bf28d7-c0be-426d-a737-d4e54856d9f4]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.265354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:19.265354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:19.265354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:19.266951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:19.268704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d42a5f9-ae42-44fd-9b1b-7315ab030872]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.270768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf61da9f-a373-4c69-b025-cede9f5e4dc6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.272615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4e04aca-6874-47c2-b57f-54fa3d0ad75d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.300428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89005813-e68e-46fd-bf62-00af8a28e18f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 307531, 'reachable_time': 18996, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142707, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.303036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:19.303723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19c867e6-e0d4-4f0f-bf5b-07100620c5ba]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:19.306043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:15Z", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:18Z", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:19.310295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:19.337953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id 01635dd2-a624-4888-bb4b-9c19d82d5c33. Wait time elapsed: 4.633 Jul 01 09:36:19.337953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id 01635dd2-a624-4888-bb4b-9c19d82d5c33. Wait time elapsed: 4.634 Jul 01 09:36:19.340886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:20.908099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b6dd1b6d-1973-4660-b4df-5c2ccb32e72e admin admin] Got routers updated notification :['b361ccc3-7577-4438-b123-f16e00901596'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:36:20.909023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 1, update_id b2894539-fbf2-4e7c-b8a0-7d742342f0cb. Wait time elapsed: 0.001 Jul 01 09:36:20.909313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 1, update_id b2894539-fbf2-4e7c-b8a0-7d742342f0cb. Wait time elapsed: 0.001 Jul 01 09:36:20.909621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:22.266997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.356s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:22.267670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router b361ccc3-7577-4438-b123-f16e00901596 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:22.268633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:22.269064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:22.269410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:22.272615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c648d0b8-fdc1-426b-95fd-d7c53ed0d082]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.295804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f97670e-b9d4-4ae0-a5e9-8499fcd0fdd2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.320966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36a887d5-31d9-43d6-b193-3034d81dfa96]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.368495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[877b4b7f-9163-4ff4-a8f7-2203315e36be]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142781, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.406502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8c1c461-0df4-4d8d-ba42-bcfab6f5cd95]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.443311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[811e4a60-6e06-44d3-af08-9ef58962878c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.460034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 7.507s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:22.460404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:22.461450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:22.461975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:22.462511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:22.465386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6df66d34-5398-491a-b25b-4ac122381e5e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.466381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58bf5d48-ddc5-4157-a93b-aaf42632214d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.486335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac26b1ec-e440-4895-9cd0-ca1895d83e0f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.494282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da1e9a5b-1b6e-46a7-ac9b-76d5abdb7a0d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.495097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:22.495592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:22.496207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:22.503829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8282e4bd-8b02-40b5-b047-a238ed29b773]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.519837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b32e9efb-d426-4e56-94f8-41bf12a19af6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 307862, 'reachable_time': 35704, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142791, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.520950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:22.521653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:22.522236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:22.522790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:36:22.523456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:22.523885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:22.524377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '7e3649ba-21e8-45b7-815d-c38c69b13308', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:66:5d:8b', 'admin_state_up': True, 'status': 'DOWN', 'device_id': 'b361ccc3-7577-4438-b123-f16e00901596', 'device_owner': 'network:router_gateway', 'standard_attr_id': 2039, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.98', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:36:19Z', 'updated_at': '2022-07-01T09:36:21Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-7e3649ba-21), ns(qrouter-b361ccc3-7577-4438-b123-f16e00901596) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:36:22.538248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[562334ed-3b49-4357-9a32-005a4afccb2c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142792, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.556273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-7e3649ba-21 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:22.556897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54d56452-111d-4da2-9769-f5afc6d992b9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.565094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2acad130-06a5-4d60-8854-fba3f4d739d5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.566691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-7e3649ba-21, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:22.567185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:36:22.568027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-7e3649ba-21, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:22.568838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-7e3649ba-21, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:22.569311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-7e3649ba-21, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:22.569778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-7e3649ba-21, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:22.570191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-7e3649ba-21, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7e3649ba-21e8-45b7-815d-c38c69b13308', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:66:5d:8b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:22.583318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f43486f6-73fa-4143-97b5-b1b5e8cd6d2b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.627277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f678e76-0df4-47e8-8e5d-48ba8494d2e0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.664369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8761eb6f-6f3b-4407-933a-7664a7ca3653]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.667459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3c87d230-4788-4f3c-be46-11583e08d4cd]: Network interface qg-7e3649ba-21 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:22.667459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:22.667459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:22.667459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:22.667459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:22.667459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:22.667459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:22.667459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:22.667459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:22.667459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:22.667459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:22.667459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:22.667459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7e3649ba-21 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:22.668538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c87d230-4788-4f3c-be46-11583e08d4cd]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-7e3649ba-21 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.669144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:36:22.669752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:36:22.686180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcb53f70-a625-4087-9c2c-f3bff64481de]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.695958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e12ccc03-b955-4109-bccf-23873906ffe0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.729501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7b57b57-7773-4302-a482-9d64929c31c7]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.730057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:22.730494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:22.730761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:22.731037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:22.742239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-19ee6a13-b8b3-44a6-8f51-9b59882dbc6a admin admin] Got routers updated notification :['b361ccc3-7577-4438-b123-f16e00901596'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:36:22.751885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:22.752176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[391e3cd2-cac2-4a38-a574-28510c43a746]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.759551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6475846d-0a24-4d2c-a41e-52e50abee5f2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.760619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:22.760949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:36:22.761673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:22.762509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:22.762899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:22.763280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:22.763685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:22.820389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eaf4aad4-09d1-4785-b60f-48152fec7685]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.861053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[caaaaec9-4168-452d-ae4f-8929478aece1]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:22.861053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:22.861053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:22.861053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:22.861053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:22.861053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:22.861053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:22.861053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:22.861053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:22.861053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:22.861053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:22.861053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:22.861053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:22.862332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[caaaaec9-4168-452d-ae4f-8929478aece1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:22.863436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:22.866465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:22.869475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8693e69d-c71e-4343-83ab-4a82696b3952]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.870472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:22.870890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:22.871335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:22.871900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:22.877513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78b41d60-637b-4288-82ff-e69960fb2639]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.880116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c00ced73-fbc1-440f-b7ba-361a8e100525]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.880561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80e52fb1-7172-4b9b-a280-f856ea80c0fa]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.882121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4dc3ee0-35c4-477b-861b-43b2dc0d065e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.918061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[290063be-0b22-44e4-b46a-e50d384b4a3a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.920273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2bd5f6c1-b8fe-47a2-b13b-bb2cfa67323f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 307880, 'reachable_time': 43456, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142823, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.929586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:22.929851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6d9edc6-cd0d-4d5b-b909-603f7ab913b2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:22.930871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:20Z", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:22.934480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 431852e6-8d33-429b-a28c-5c863668fe44. Wait time elapsed: 11.944 Jul 01 09:36:22.956303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 431852e6-8d33-429b-a28c-5c863668fe44. Wait time elapsed: 11.945 Jul 01 09:36:22.956303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:23.108401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ec84f26-01a9-4414-9013-6bb2e040be74]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.109204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:36:23.109766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:23.110478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:36:23.111089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" released by "process_external" :: held 0.588s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-7e3649ba-21 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7e3649ba-21 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7e3649ba-21 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:23.112308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7e3649ba-21 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7e3649ba-21 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:23.117607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:23.122024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f421e35-6b7a-41d6-809b-a03cbd4133d2]: (4, ['qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.122943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:23.123614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:23.123824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:23.124278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:23.127210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2e8da5f-fb35-4526-8276-fbe647209b57]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.128897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30e9b6a5-4ba2-4ab7-9a51-a0b7717cf2be]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.129884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24b11493-1306-4589-b673-80f12a2e86e9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.162300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9702aed2-fcfa-4039-b8a1-168789fb4608]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 307862, 'reachable_time': 35704, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142825, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.169099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:23.169464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1706986-f0d4-487d-bed3-4238c3b553bd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7e3649ba-21 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7e3649ba-21 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:23.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-617448100-router", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98" Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:18Z", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:20Z", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:66:5d:8b", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2039, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:19Z", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 1, update_id fa412ee9-326b-499a-bc1e-2d694676db05. Wait time elapsed: 0.430 Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 1, update_id fa412ee9-326b-499a-bc1e-2d694676db05. Wait time elapsed: 0.430 Jul 01 09:36:23.174785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:23.261301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.926s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:23.263131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:23.263131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:23.263832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:23.264272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:23.266060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b30d9b45-c278-491e-b444-a644e5ea16a3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.303409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30d94c14-dfaf-424e-a551-7b34f97b90fd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.320462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1a7a71b-5e82-4204-884f-b70fbc723476]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.359041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[774502d2-0dcb-4e1e-8a10-9a7402b03451]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142828, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.385470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24ac2603-a7c5-482a-be55-82d27a1fd455]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.415521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00739fda-1c94-42f4-92b8-03a1ffbcf20f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.464707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f8e5464-0abb-447d-bb63-53c9032cba85]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.489197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a30190c4-ec5f-4a38-bf85-ca94b62b99b8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.491439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:23.491790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:23.492150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:23.492505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:23.513962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:23.514567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3751f5eb-57c6-424e-ae74-c3b3a02c9d83]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.519740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d86f5184-6c9c-4ef7-b2d7-c9fd6c16c075]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.521751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:23.581467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:23.582391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:23.583014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:23.583704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:23.584453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:23.640542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44f8c10b-a312-4b05-968b-45a23b51ffac]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.687205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cc57dee2-31ad-4ff5-9890-fa69ccb697d8]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:23.687205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:23.687205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:23.687205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:23.687205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:23.687205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:23.687205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:23.687205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:23.687205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:23.687205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:23.687205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:23.687205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:23.687205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:23.689386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc57dee2-31ad-4ff5-9890-fa69ccb697d8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:23.690602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:23.693698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:23.698421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75a27db5-7d89-4bdd-ad26-a860d4130085]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.699569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:23.700010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:23.700359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:23.700802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:23.702514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9128392-52a9-4d8f-b469-3bab44ff9bf5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.705240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38a4b2fe-9018-4ed5-a98c-0bdce99df547]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.706477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90bd1b14-dc0d-4f6d-a9e7-007972eece07]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.730124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6214a95e-001f-41c2-8d76-ae75976f022e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 307962, 'reachable_time': 35560, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142846, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.739068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:23.739660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58806f61-94a9-4e30-b1a6-b48a5490af97]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:23.740900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:18Z", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:23.744305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id c9419d93-e4b7-4bc6-b96e-6b06ff675fa5. Wait time elapsed: 6.676 Jul 01 09:36:23.756873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id c9419d93-e4b7-4bc6-b96e-6b06ff675fa5. Wait time elapsed: 6.676 Jul 01 09:36:23.756873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:25.150483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.976s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:25.151065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router b361ccc3-7577-4438-b123-f16e00901596 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:25.151809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:25.152301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:25.152892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:25.154277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4e03f96-835e-405e-9276-3950234ef1d7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.179332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2232f17a-0f90-4875-8ffd-19f18dc2b01e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.210452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[113cb30f-53b2-409b-b620-5b2c04b69fcf]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.262140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f9e0746-f40f-4e56-b8b2-382a306bbf21]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142851, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.292482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f26ee110-14ed-4d4b-87ec-2517a85014d9]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.316960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66abb66c-1e2c-4645-8602-1d0a7a3fbe8b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.357596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b4e3c25-530f-4efc-86b5-15fb888e06e6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.394899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6dc12d8f-b756-48e3-8b19-b2fbf0022899]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.395979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:25.396259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:25.396530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:25.397372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(f562c8af-67d2-4171-8236-1336b7353fdf) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:25.437041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:25.437164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[644be7ae-71ff-4aca-885f-f8813acd96c4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.440709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c353262-5fc1-4859-806e-63d633e99f1b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.442085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-f562c8af-67, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:25.442707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:36:25.443859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-f562c8af-67, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:25.444732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:25.445383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:25.446129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:25.447232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-f562c8af-67, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f562c8af-67d2-4171-8236-1336b7353fdf', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:56:46:6a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:25.508648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b53d9aa-5986-44e4-8256-775330d32a9f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.548845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6b9e7537-0925-4524-b4d9-cdda8fdde1f7]: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:25.548845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:25.548845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:25.548845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:25.548845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:25.548845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:25.548845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:25.548845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:25.548845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:25.548845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:25.548845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:25.548845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:25.548845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b9e7537-0925-4524-b4d9-cdda8fdde1f7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:25.549999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:25.552006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:25.554420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69ee5052-fda0-4b38-8d20-82a63da78d01]: (4, ['qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.554420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:25.554420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:25.554708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:25.555242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:25.556407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1470065a-4fa3-428b-9ca9-34b36b2e8000]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.557504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1ec4b7b-8f4c-4159-a8b3-22c452685e56]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.558797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[728fe4f5-f77b-4715-93cf-bf6f8c7cb843]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.585888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14ec67b7-ff2a-44c7-a1a2-a1344ea64961]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 308152, 'reachable_time': 42272, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142869, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.588621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:25.588901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[977fea38-9b2f-4d1f-9924-771a649aa0ae]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:25.589927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-617448100-router", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98" Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:18Z", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:66:5d:8b", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2039, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:19Z", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f562c8af-67d2-4171-8236-1336b7353fdf", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0cb463ae-0884-401c-b0c4-7519eb64a440", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:56:46:6a", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2040, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:21Z", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:24Z", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:25.593210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id b2894539-fbf2-4e7c-b8a0-7d742342f0cb. Wait time elapsed: 4.683 Jul 01 09:36:25.608523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id b2894539-fbf2-4e7c-b8a0-7d742342f0cb. Wait time elapsed: 4.683 Jul 01 09:36:25.608523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:26.349376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.758s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:26.350407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router b361ccc3-7577-4438-b123-f16e00901596 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:26.354367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:26.354893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:26.355332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:26.356580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d6eecc2-1ed5-4af6-9a4c-83c73a42cf9b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.368470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f4c6bbe-6392-4794-b148-17d4a016a717]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.393524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[254abfab-b29e-487a-9413-c3d15744003e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.440689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2970f511-021f-4c5b-bf05-0005ccf2ad78]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142894, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.463201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2be2797-d1c0-4aa9-b7ac-076d60c6e251]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.510046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da324530-4125-4d59-a01f-da93b0e2f35f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.534324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce6b6333-d80f-442b-8a1a-add5d456955c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.577079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddb28fb2-40fd-424c-ba42-0e54824733cf]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.577881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:26.578384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:26.578747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:26.579735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(f562c8af-67d2-4171-8236-1336b7353fdf) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:26.608690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:26.609229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[583ea635-3862-4d29-8f82-00afa34606b8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.613406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c7b0146-dde8-4e7f-9be1-473817eb7c99]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.614931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-f562c8af-67, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:26.647687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-f562c8af-67, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:26.648547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:26.649796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:26.650478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:26.651060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-f562c8af-67, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f562c8af-67d2-4171-8236-1336b7353fdf', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:56:46:6a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:26.704192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ad5577a-8ea0-48ba-8c0b-16ab83eff1bb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.760714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a14971fe-3299-4721-bf02-660de2fc05fb]: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:26.760714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:26.760714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:26.760714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:26.760714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:26.760714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:26.760714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:26.760714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:26.760714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:26.760714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:26.760714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:26.760714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:26.760714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:26.761820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a14971fe-3299-4721-bf02-660de2fc05fb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:26.761921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:26.763840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:26.768230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8d950b2-65ed-4b25-9610-ac17f4476477]: (4, ['qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.769079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:26.769534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:26.769728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:26.770304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:26.771238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e65c105-c831-49dc-a243-f5038f277e0e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.772463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c44e55f2-bc45-487d-a0f4-5aad62c17f8b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.776476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[842985ed-93be-4202-9771-666b9f97a9cf]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.815406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b2d6486-a7a8-4ff0-8ad0-694ba00a49a2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 308270, 'reachable_time': 15572, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142914, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.822839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:26.822839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1bce5066-d676-48b3-acf7-c200157f6797]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:26.824044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-617448100-router", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98" Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:18Z", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:66:5d:8b", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2039, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:19Z", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f562c8af-67d2-4171-8236-1336b7353fdf", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0cb463ae-0884-401c-b0c4-7519eb64a440", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:56:46:6a", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2040, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:21Z", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:24Z", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:26.827263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id fa412ee9-326b-499a-bc1e-2d694676db05. Wait time elapsed: 4.083 Jul 01 09:36:26.841397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id fa412ee9-326b-499a-bc1e-2d694676db05. Wait time elapsed: 4.084 Jul 01 09:36:26.841397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:26.930876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.998s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:26.931150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:26.932136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:26.932465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:26.932747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:26.933990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[367c32e7-9643-46aa-a710-1654433ca823]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.946949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9046bb77-9efe-4546-86b6-a9dbc0fd528c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:26.970033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab7a9054-e7a2-49b6-89ec-da33ea911824]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.003003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[807f5f55-702a-4560-88a8-96e46b8580c4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142917, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.024422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b69cfd81-7a82-4441-94f8-6d02384482e0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.038114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e900233b-ebb5-4f66-be47-cd433abe11ea]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.060289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3bb0a49-a39a-4cf6-b9a7-ed809a20d0ca]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.092431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e82ac1d0-ecdb-49e6-b69b-76f334fb493a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.093255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:27.093691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:27.094809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:27.095980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:27.117832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:27.118350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcc1a8e2-c645-4e33-8165-742969de962a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.124491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[afe5d52f-6b5f-4430-9e33-e0aceb096919]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.125814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:27.167519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:27.168354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:27.169472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:27.169951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:27.170406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:27.227696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07e36f44-e34f-4c8e-8b27-d8d70c206722]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.264510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d15e6299-1dbf-4485-84b5-08853600d51e]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:27.264510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:27.264510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:27.264510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:27.264510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:27.264510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:27.264510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:27.264510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:27.264510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:27.264510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:27.264510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:27.264510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:27.264510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:27.265926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d15e6299-1dbf-4485-84b5-08853600d51e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:27.267869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:27.269559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:27.273111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[390dc547-30b9-408c-968d-034617462074]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.274008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:27.274427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:27.274735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:27.275574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:27.276528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea9cb1d6-441b-49c2-b1e5-55cd0cff8125]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.278111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c37a93f-36f4-487a-82b2-858813eb04bf]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.279237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a39ac68-db4d-4ac6-b6af-2e2c46cf3f94]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.301533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4b8e8f4-d573-4568-9707-5d683e4d2ee6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 308326, 'reachable_time': 15400, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142935, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.307169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:27.307614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a23c380-b6fc-45ea-9914-a51d839139e9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:27.308988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:20Z", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:27.311931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id c9266c6b-246e-4b1a-b825-2e8237beecb6. Wait time elapsed: 12.740 Jul 01 09:36:27.324402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id c9266c6b-246e-4b1a-b825-2e8237beecb6. Wait time elapsed: 12.740 Jul 01 09:36:27.324402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:28.293832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.983s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:28.294502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:28.296274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:28.297092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:28.298982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:28.300318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[033f06ed-e7c1-486c-97d0-2194bd085fbb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.321055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d60c5f40-edc0-4ade-a536-1830784510d0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.358373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7dd21cbf-377f-473c-bbe3-7fb724133169]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.406458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc139e7e-696e-4acd-9969-1f002c872272]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142938, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.431025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22f3499f-3d5a-4762-bd67-184dd82b5d56]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.456131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eefd3086-82be-444b-98cd-1d576ec02d8e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.482329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4948471e-9b18-4a12-bc0e-69a74527b87a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.508323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3552d87b-9539-4b1b-bac5-b3f860563183]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.509257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:28.509630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:28.509986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:28.510475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:28.528957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:28.529590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0f65290-8977-400a-8595-aa164c9756b7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.532821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6fda06f-93d4-4b61-ab9b-ece027739850]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.533753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:28.563055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:28.566537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:28.567167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:28.567826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:28.568404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:28.607784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55c03ad5-bc31-4149-ae38-0518af30283e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.650572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5c497794-84b2-400d-aab3-c0d496708584]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:28.650572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:28.650572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:28.650572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:28.650572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:28.650572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:28.650572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:28.650572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:28.650572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:28.650572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:28.650572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:28.650572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:28.650572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:28.652088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c497794-84b2-400d-aab3-c0d496708584]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:28.653254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:28.657847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:28.662535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a5b742e-dcb8-4079-b51b-15c6e2ee3386]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.663480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:28.663862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:28.664121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:28.664523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:28.666427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef8229ba-405f-4f9f-9a62-edf9b8041367]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.667663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5363259-70a9-4a19-b2b4-ad764e9b2d86]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.668790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbb2c46a-9489-4ee1-bec3-eac83f0cedf7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.692435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52e95222-47cf-44c2-92d4-682f6d6917ac]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 308466, 'reachable_time': 15120, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 142959, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.700192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:28.700660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4779f31e-ba88-425d-b650-06750d561e41]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:28.702007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:20Z", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:28.706875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 431852e6-8d33-429b-a28c-5c863668fe44. Wait time elapsed: 17.718 Jul 01 09:36:28.727012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 431852e6-8d33-429b-a28c-5c863668fe44. Wait time elapsed: 17.718 Jul 01 09:36:28.727012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:28.919087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.174s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:28.919833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:28.921014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:28.921513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:28.924074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:28.925492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6740276-913f-4e1a-a505-e81aae0da417]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:28.960115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[074b3dac-ecfe-4961-8d85-a97d2aaab329]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.002499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52c68437-848e-41d6-a981-8494db4a9e37]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.083781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c65e4db-8811-40d8-9c89-b0659f66b094]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 142985, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.121084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40d82376-fcdb-4484-b31b-699e8253f849]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.146238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3eb77b58-ecba-4059-81b3-ce057d05577a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.186292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[362abd94-658d-40fe-af86-af939059d564]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.221557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed48f6cf-9549-41a2-a280-b78b6ec72da1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.222232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:29.222569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:29.222908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:29.223179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:29.248487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:29.248775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f289e70d-7766-488f-b67a-f2a3a874f92a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.252164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9998215-a4f5-468a-9960-81c306c98ea3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.253087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:29.286603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:29.288108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:29.288714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:29.289745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:29.290344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:29.337518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a7fbf1b-eb3e-4f6c-bdf9-9f39e3e4763b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.393340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6aae1977-fa3e-4675-891b-d60ad539f77a]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:29.393340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:29.393340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:29.393340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:29.393340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:29.393340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:29.393340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:29.393340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:29.393340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:29.393340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:29.393340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:29.393340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:29.393340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:29.395209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6aae1977-fa3e-4675-891b-d60ad539f77a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:29.396434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:29.398083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:29.400299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7074f672-7d72-4abc-950c-aed3d5cfdeec]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.401057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:29.401322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:29.403600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:29.403966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:29.410190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e81f961-0421-4cf0-b5d9-1e1eab81fe35]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.412295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[251d72b5-23a7-4d3c-a396-7db8b27e0168]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.415425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1c7b57b-0407-4bd6-9e23-33e5e926ef17]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.456868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffd3420f-41c8-40f3-85c0-7e1c0dc28a91]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 308534, 'reachable_time': 22972, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143016, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.465940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:29.466186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11b978df-be83-4990-8243-22bd3a9e584c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:29.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:25Z", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:29.472493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:29.488085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id 01635dd2-a624-4888-bb4b-9c19d82d5c33. Wait time elapsed: 14.775 Jul 01 09:36:29.488085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id 01635dd2-a624-4888-bb4b-9c19d82d5c33. Wait time elapsed: 14.775 Jul 01 09:36:29.488085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:30.393810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.920s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:30.394447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:30.395019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:30.395342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:30.395645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:30.397017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b35970c-b932-4f43-b129-dd0e8d940a6c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.447570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c99b8cc7-95d8-454a-8728-bba2cbdc82da]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.485994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8af8c960-9614-4fbb-b20a-7260ba78504e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.533066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6e0285c-f0e0-42f7-8549-58e6fb61e219]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143049, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.574414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2b12506-3001-49c6-a6eb-f55bc368b4e4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.622942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c185fda-306b-4e4b-8991-b3ff412b5f31]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.657385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[916c6a63-de6d-4952-91c1-2ac8e3b5b07f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.694041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48f1df17-f43b-4608-a355-fd4ef924aa27]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.694593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:30.694863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:30.695316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:30.695787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:30.738047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:30.738047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75a63449-ac2d-4fe6-aa31-77e6908bb96a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.738047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b147eb0-d32d-4347-add4-cd7a07b14b1a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.738047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:30.773248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:30.774178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:30.775073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:30.775725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:30.776337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:30.821106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[997104c0-27a0-405b-b514-98d3208477ed]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.864457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bcd9173f-8758-4d98-b612-7b877192affd]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:30.864457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:30.864457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:30.864457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:30.864457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:30.864457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:30.864457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:30.864457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:30.864457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:30.864457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:30.864457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:30.864457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:30.864457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:30.866494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcd9173f-8758-4d98-b612-7b877192affd]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:30.867717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:30.870760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:30.876016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf427959-718c-4a91-89c7-98042701921b]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.876882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:30.877316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:30.877640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:30.880604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:30.882014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e10bfdf-31f2-4f8c-a2f5-a4751696ff6b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.883476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30b21736-9be7-4804-8871-48d4ecfde43d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.885666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfe0a2f5-ea09-4e8e-bf39-21ee1e630181]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.930642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efe52e03-bfe6-45e4-ab0e-7dbcbbc656da]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 308679, 'reachable_time': 25724, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143067, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.943074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:30.943343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d1a808e-3443-4792-b3bb-5ceb548524bb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:30.944645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:25Z", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:30.948118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id c9419d93-e4b7-4bc6-b96e-6b06ff675fa5. Wait time elapsed: 13.878 Jul 01 09:36:30.964551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id c9419d93-e4b7-4bc6-b96e-6b06ff675fa5. Wait time elapsed: 13.879 Jul 01 09:36:30.964551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:31.862503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.035s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:31.863103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router b361ccc3-7577-4438-b123-f16e00901596 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:31.865104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:31.865767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:31.866191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:31.872681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[930b026b-2052-4b99-931d-4cedba6250b6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:31.894803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77194f3e-f1b7-4cd4-9b20-604e96a8f6a7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:31.921944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4e4053c-5bb8-41cc-a600-94dba49525d7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:31.965994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97f5c2ab-3ac3-450f-b825-15892a612754]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143071, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:31.989200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e677f51-eb07-4a30-adc8-7d11f8fce22b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.010948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9090680-c01c-4fbe-8af5-5d4b943cb7c8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.034084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8eacf6cd-5d41-4336-882a-c16ba0dec24c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.054883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7fe0b2ca-f5ac-403e-9840-e8dba2bdf725]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.055763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:32.058617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:32.058961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:32.059455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(f562c8af-67d2-4171-8236-1336b7353fdf) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:32.078093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:32.078611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ceac531e-dbe9-4866-801d-a939e39f6745]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.083889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[494b31ed-0f83-4a13-9fe7-4a0a2463639e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.084845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-f562c8af-67, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:32.114449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-f562c8af-67, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:32.115439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:32.116017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:32.116405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:32.116786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-f562c8af-67, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f562c8af-67d2-4171-8236-1336b7353fdf', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:56:46:6a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:32.164610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[294369bd-be8f-4ce3-9ff3-70293a2c3334]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.201314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[98f689d6-9a13-4d70-b203-0a11298b0e94]: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:32.201314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:32.201314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:32.201314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:32.201314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:32.201314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:32.201314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:32.201314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:32.201314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:32.201314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:32.201314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:32.201314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:32.201314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:32.202594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98f689d6-9a13-4d70-b203-0a11298b0e94]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:32.202937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:32.205207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:32.208157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51c40662-9c4f-48d2-b317-9a691075d8b3]: (4, ['qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.208157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:32.208157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:32.208157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:32.208157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:32.209214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88572993-9ff9-45d1-be44-469bb6329ef4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.210252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e7db0f8-81cd-48aa-a69d-c28ad5f91dc2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.211182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff24ec89-3938-46fd-b124-089653262e6c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.233465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f05df30-0f46-42d4-bc2a-094cc0329c55]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 308822, 'reachable_time': 23076, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143090, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.239316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:32.240028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db43ae14-c287-4923-9bad-5fbff0defde6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:32.242099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-617448100-router", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98" Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:18Z", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:66:5d:8b", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2039, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:19Z", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:28Z", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f562c8af-67d2-4171-8236-1336b7353fdf", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0cb463ae-0884-401c-b0c4-7519eb64a440", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:56:46:6a", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2040, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:21Z", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:24Z", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:32.245238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id b2894539-fbf2-4e7c-b8a0-7d742342f0cb. Wait time elapsed: 11.335 Jul 01 09:36:32.260473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id b2894539-fbf2-4e7c-b8a0-7d742342f0cb. Wait time elapsed: 11.335 Jul 01 09:36:32.260473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:33.061172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.817s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:33.062309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router b361ccc3-7577-4438-b123-f16e00901596 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:33.064603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:33.065321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:33.066480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:33.067814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fab43df6-9a6e-4cd7-ba1c-df60cd5b8b25]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.088139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec2f073d-7c70-4e3f-b96a-c1d4f38aff35]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.112490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d842159b-28d8-4046-b49e-5bf9deb86cb5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.161323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e452908f-e9de-4efa-9161-8437f68f32f0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143094, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.190957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07372df6-35ff-460d-8bdd-88e7e6366568]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.214876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1118c922-31d7-480a-be7e-089a2de73c0b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.231592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.516s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:33.231592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:33.231592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:33.231592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:33.231592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:33.231592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[132e986e-0c07-400c-ae25-3d6bb7199f7b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.252698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2015dd58-bbd4-49b9-b288-916d5488b48c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.253267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5d94c65-4d3c-4019-93e8-eb29090872fb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.276891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[489bbd7c-6f3e-44fd-8e7f-a0a929434f3d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.277505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:33.278040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:33.278309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:33.278831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(f562c8af-67d2-4171-8236-1336b7353fdf) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:33.287561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[197d470a-1ae9-4f39-971b-37489f5d76fe]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.293752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:33.294230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d17146dd-14f2-45e9-9b1e-435285ca2862]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.298875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35eba455-e6c2-49ba-a05c-cde61c31e004]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.301011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-f562c8af-67, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:33.327083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59bba1b7-3e3c-4a1f-b8e5-b5058dd29af2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143103, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.351523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-f562c8af-67, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:33.352274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:33.353054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:33.353556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:33.354037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-f562c8af-67, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f562c8af-67d2-4171-8236-1336b7353fdf', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:56:46:6a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:33.371681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[476f8450-b68c-4ea1-8525-54202ae4824c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.409655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[553d219c-7de7-4fea-95d5-ddd9ae17d50f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.436572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[281cb4e7-710d-4e25-acf8-0934af68970d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.443992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a431a4d5-63f9-431a-95d0-9c04c752c9a6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.472982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a8df01d-4815-484d-98db-fc309f087fe2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.473846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:33.474125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:33.474570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:33.475687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:33.491816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9a3b7f4f-c9ba-44ce-9c7f-d9fb1c8dc02a]: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:33.491816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:33.491816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:33.491816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:33.491816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:33.491816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:33.491816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:33.491816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:33.491816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:33.491816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:33.491816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:33.491816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:33.491816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:33.493099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a3b7f4f-c9ba-44ce-9c7f-d9fb1c8dc02a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:33.494267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:33.496926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:33.499908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b623fd9-99d0-49cf-84ec-e9667a0afde1]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.501973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:33.502423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:33.502741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:33.503175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:33.503796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:33.504134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23f69d99-e380-4d23-82ba-564d2ce2b639]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.504527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bdfb1167-5a04-4277-8e1f-596da736c8ed]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.509036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65d30cc0-1850-4bac-b50d-b314ddd683f2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.509328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6e8334f-6a4a-4232-a101-b24404a42e46]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.511159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:33.521362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88af8ac3-f136-42f3-a60c-f2b06ad0bfbe]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.545469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:33.546534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:33.547312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:33.548137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:33.548739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:33.561068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[237e96ff-d5b9-4002-9056-aef3ada86423]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 308942, 'reachable_time': 31476, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143123, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.572979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:33.581466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[489add62-529b-4572-8c89-8d9016aef054]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:33.583314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-617448100-router", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98" Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:18Z", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:66:5d:8b", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2039, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:19Z", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:28Z", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f562c8af-67d2-4171-8236-1336b7353fdf", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0cb463ae-0884-401c-b0c4-7519eb64a440", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:56:46:6a", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2040, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:21Z", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:24Z", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:33.587517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:33.605297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id fa412ee9-326b-499a-bc1e-2d694676db05. Wait time elapsed: 10.853 Jul 01 09:36:33.605297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id fa412ee9-326b-499a-bc1e-2d694676db05. Wait time elapsed: 10.853 Jul 01 09:36:33.605297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:33.618875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9027e364-d910-4a6d-9e0e-e009f5e76ef8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.655737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[54269119-db67-4497-9c17-a0a02f9561a4]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:33.655737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:33.655737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:33.655737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:33.655737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:33.655737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:33.655737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:33.655737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:33.655737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:33.655737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:33.655737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:33.655737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:33.655737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54269119-db67-4497-9c17-a0a02f9561a4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:33.656945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:33.658994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:33.663333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c11b7920-ffa6-4d7f-9a9c-e68e9403d54e]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.664277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:33.664659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:33.665733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:33.665733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:33.667454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bca67e38-10f7-4ffe-8a64-00ef078b0e00]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.668558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4c12f2d-86ed-452e-aab6-4dd5d360815c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.669479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[206e341d-8d73-456b-a164-a639fb23eb12]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.692121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9227b39c-23d3-4557-b6bf-e3ff3174bc01]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 308958, 'reachable_time': 41976, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143136, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.699925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:33.700336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9842f137-7100-4abf-a394-f8ae7e27e29f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:33.701470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:20Z", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:33.705600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id c9266c6b-246e-4b1a-b825-2e8237beecb6. Wait time elapsed: 19.134 Jul 01 09:36:33.726090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id c9266c6b-246e-4b1a-b825-2e8237beecb6. Wait time elapsed: 19.134 Jul 01 09:36:33.726090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:34.642803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.936s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:34.643557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:34.644268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:34.644696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:34.645070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:34.646285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[991f51e4-4831-43a9-9724-4b19fd501855]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.662188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.715s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:34.662740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:34.663942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:34.664364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:34.664742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:34.665820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7eae9975-4c27-44d7-81b2-11fc3c394039]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.671195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a43858b3-0e55-42fa-80b1-f39c1c656c59]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.691965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8f57be5-48cb-4875-823d-f0ccd6aa6c80]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.696477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8d50896-f6d5-4fd0-83ac-a870f6d4c473]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.716322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f351c8ee-9610-414f-b3cc-a3fdce381cad]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.747767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f65ee14-e154-4fe1-9a1d-1cdfae74d1b1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143141, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.765242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fd8efba-6afb-428a-9922-d5941e704c6e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143143, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.779701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[332308eb-d233-407c-a935-2eb21e685193]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.788338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da5d6376-c1de-4dd9-a849-1d7f957a80b2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.793902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08bd39c2-c4a5-47b3-9c35-55c1bc418c4a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.802725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60586bcb-8ce3-47a8-8a32-cfb6b2f64227]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.823178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54bdcc3d-2888-4333-bb76-dbc967e70449]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.829681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa42aee7-024e-4251-a3de-adf298b5c758]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.848690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[352bed20-3639-447d-a725-d708042480c5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.849562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:34.849990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:34.850404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:34.850849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:34.860327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[423b3f5d-fa30-4168-90d2-7c2adc5a36d7]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.861341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:34.861818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:34.862247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:34.862689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:34.883223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:34.884556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[208e1595-176c-4f7e-aa92-22f00511994c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.889593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8d3be07-8683-49e5-beeb-0ebeada9cee3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.891551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:34.908988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:34.909363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f2d15bb-6aff-4144-baae-4de9038522de]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.914545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18d85db8-57e9-445e-8461-ccbf36803c36]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:34.921354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:34.972691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:34.973555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:34.974163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:34.974787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:34.975183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:35.007953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:35.008980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:35.009730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:35.010301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:35.012257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:35.034610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d34c572e-78c4-4829-8beb-f711ac371476]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.092942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26e44a49-c0b6-4525-b462-8ed8e984aef1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 3141474383, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.120995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[efd9c656-36e9-4d62-80ed-30195eacd3c7]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:35.120995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:35.120995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:35.120995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:35.120995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:35.120995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:35.120995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:35.120995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:35.120995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:35.120995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:35.120995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:35.120995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:35.120995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:35.122919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efd9c656-36e9-4d62-80ed-30195eacd3c7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:35.125932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:35.128934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:35.138922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e5c02c7d-14c5-4cf0-9931-68daeec2f440]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:35.138922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:35.138922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:35.138922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:35.138922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:35.138922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:35.138922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:35.138922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:35.138922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:35.138922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:35.138922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:35.138922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:35.138922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:35.140036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5c02c7d-14c5-4cf0-9931-68daeec2f440]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:35.140975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:35.143027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:35.145707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13bf4ac9-a2ba-40a0-978e-12d763ba7c9e]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.145707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b694ed5-7e32-45d3-acaf-c34b2e16699f]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.145707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:35.145707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:35.146149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:35.146149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:35.146314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:35.146743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:35.146894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:35.147226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:35.147860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8fb87a2-394b-497f-9faf-a48a5d43bac8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.148175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d64837a7-9ee3-4bb2-89d8-76a1c009ab8c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.149175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5536a359-315b-42e1-8308-2d73428347f0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.149490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3fcb299-96e0-4d52-9d2c-497b7ac1346a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.153443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5de4e41-a31c-4d2b-9022-e26a723c0e02]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.155274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a59c8c79-1e45-46f6-95e3-6b8fac9e09fa]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.187954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14adac25-8e1b-41d9-abc5-4cad93af26a2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 309100, 'reachable_time': 41104, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143183, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.191243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:35.191687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9edb75ab-53f3-4216-b4cf-f87c9d231aee]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:35.192889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:32Z", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:35.196808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 431852e6-8d33-429b-a28c-5c863668fe44. Wait time elapsed: 24.208 Jul 01 09:36:35.208401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 431852e6-8d33-429b-a28c-5c863668fe44. Wait time elapsed: 24.208 Jul 01 09:36:35.208401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:35.208401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c4ebbbf-ec5e-4776-8f71-055846b12f67]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 309102, 'reachable_time': 41124, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143184, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.220403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:35.221591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96503c05-cb7d-4acb-bff8-67daf219cbfe]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:35.222874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:30Z", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:35.226902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id 01635dd2-a624-4888-bb4b-9c19d82d5c33. Wait time elapsed: 20.526 Jul 01 09:36:35.263817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id 01635dd2-a624-4888-bb4b-9c19d82d5c33. Wait time elapsed: 20.527 Jul 01 09:36:35.263817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:36.058041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.832s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:36.058693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:36.059275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:36.059867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:36.060185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:36.061252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[282221fc-c345-422e-8a0b-592a7473a545]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.080562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c470bad1-f0c4-4573-805f-6b95d0fa6dd7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.100920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eaf96b73-f60c-4daf-942d-607669bc1d02]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.132065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5b27738-1d36-477e-9b4b-18050036da0e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143200, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.158671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62c9fde9-8ba7-4497-9dda-d02c26771110]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.186145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[484cc5b1-aa76-42cb-96dc-2ad9819661ee]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.210442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b01079f-d2b3-4a3b-a208-2ecf44f684da]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.238861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4ab5516-a04b-43a8-b8c7-3dd0e2fb2040]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.239528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:36.240355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:36.240799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:36.242040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:36.268823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:36.269704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f37beca-117d-440a-ad32-e1159434766c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.273203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb842e2a-db69-47fa-871f-d0f40dc49251]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.274249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:36.305193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:36.306379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:36.306960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:36.307603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:36.308181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:36.352828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d79e336-2369-4761-8233-549de230eafc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.386198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[06663d75-abf3-46b3-b748-e75a80e21c1b]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:36.386198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:36.386198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:36.386198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:36.386198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:36.386198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:36.386198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:36.386198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:36.386198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:36.386198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:36.386198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:36.386198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:36.386198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:36.387252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06663d75-abf3-46b3-b748-e75a80e21c1b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:36.387422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:36.389257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:36.392263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b0a7c4e-d934-4627-97d5-5ae1c55cbfe9]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.392263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:36.392263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:36.392263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:36.392263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:36.394048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9550868-bbb1-44db-aeae-1f91d3bf4007]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.395596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50c84f29-f6d7-4489-84ca-3e33e4371c54]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.396569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dba7d8bd-8dd9-48ac-a23a-af4e4267ecfb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.421073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54ce86d8-ca3d-451d-ba75-e06b1506d5be]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 309239, 'reachable_time': 39752, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143219, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.431216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:36.431512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbd45e00-6697-4c47-8e51-6c0428520800]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:36.432547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:33Z", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:36.435514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id c9419d93-e4b7-4bc6-b96e-6b06ff675fa5. Wait time elapsed: 19.366 Jul 01 09:36:36.451073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id c9419d93-e4b7-4bc6-b96e-6b06ff675fa5. Wait time elapsed: 19.366 Jul 01 09:36:36.451073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:36.691261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.094s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:36.691884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router b361ccc3-7577-4438-b123-f16e00901596 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:36.693056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:36.693578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:36.694048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:36.695342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c966f14-f29b-430b-ab60-2a1a186c0425]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.713988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[802f4b0b-f25b-4e0d-95a9-341fbeb9a032]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.745531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14b32b40-adbd-4c5b-b993-129315b7492f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.785085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5728306e-0446-42b6-bc67-f7ba73ff9482]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143224, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.806498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d91a958d-1289-468c-9ecc-b628a18e33f0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.832834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4f15bc9-5601-4a39-9fbc-344a5c221fab]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.857272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b87b67f-a628-430c-9e48-af77e86d9cfd]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.881595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6dd7cede-d4bc-410c-a47f-ead3435edaf9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.882343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:36.882692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:36.883164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:36.883896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(f562c8af-67d2-4171-8236-1336b7353fdf) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:36.909408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:36.909702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f92ef7f-5950-4ba4-a5c3-0a04ca672a53]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.912582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87a54073-50bb-4cae-bdd4-eb7340c02f0f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:36.913917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-f562c8af-67, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:36.950685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-f562c8af-67, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:36.951443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:36.952106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:36.952496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:36.952881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-f562c8af-67, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f562c8af-67d2-4171-8236-1336b7353fdf', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:56:46:6a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:36.980880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6ac7647-108d-4ba6-b53f-1120ad7bba94]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:37.024379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[fe89a108-bf1d-45f3-b446-e3edcc0f2495]: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:37.024379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:37.024379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:37.024379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:37.024379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:37.024379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:37.024379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:37.024379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:37.024379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:37.024379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:37.024379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:37.024379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:37.024379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:37.025583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe89a108-bf1d-45f3-b446-e3edcc0f2495]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:37.025903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:37.027742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:37.030652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c51cbcae-f62d-4313-8385-4b59533027e9]: (4, ['qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:37.030652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:37.030652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:37.030652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:37.030652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:37.032444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6a76327-a9c4-4761-90fd-d0ee157648c0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:37.033401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7717c87a-bc53-4c20-bbea-45aa40a931be]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:37.034579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57870a98-afe0-4f38-a8a0-0e61f27cf8c0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:37.057226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[078c3281-7911-44b6-970f-ef139db36e55]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 309304, 'reachable_time': 21024, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143242, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:37.061223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:37.061822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efc765e9-5fce-44c1-943a-d06e4bdf6f02]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:37.063122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-617448100-router", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98" Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:18Z", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:66:5d:8b", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2039, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:19Z", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:28Z", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f562c8af-67d2-4171-8236-1336b7353fdf", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0cb463ae-0884-401c-b0c4-7519eb64a440", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:56:46:6a", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2040, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:21Z", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:24Z", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:37.066385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id b2894539-fbf2-4e7c-b8a0-7d742342f0cb. Wait time elapsed: 16.156 Jul 01 09:36:37.088546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id b2894539-fbf2-4e7c-b8a0-7d742342f0cb. Wait time elapsed: 16.156 Jul 01 09:36:37.088546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:37.944428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.880s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:37.944971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router b361ccc3-7577-4438-b123-f16e00901596 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:37.945952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:37.946463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:37.946809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:37.948849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c783c7c-008a-4a94-9ba4-846cfc68161b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:37.978914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4febe98a-af3d-4f27-99ac-043dea78c7b1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.005817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a75e86de-6fca-48d6-a109-4f384ef171d3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.044299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4783f1c4-c1d2-4ab4-b6a6-6061942b4390]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143245, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.068652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a23a3872-5ada-4802-bac2-c0be961842b5]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.094438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d4bc045-f533-4f54-96f9-a7337e76de6b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.120142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d862457a-7909-4fbc-a724-e911db08037e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.134029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.937s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:38.134371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:38.135760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:38.136152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:38.136658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:38.138688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51bdb649-8c5b-4522-98eb-89e9abb22cae]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.144258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90db5d62-0a5a-4804-9456-0a5c113da753]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.144888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:38.145233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:38.145571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:38.145924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(f562c8af-67d2-4171-8236-1336b7353fdf) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:38.155777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35dae7a4-f129-4a6c-b5e1-1b6e49712525]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.177384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:38.177759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a6111dd-c291-470c-bded-cb504bf22538]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.181120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f75cddc-1ede-4074-8ac1-6ccd9881746e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.182274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-f562c8af-67, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:38.198463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4278efd3-1aa7-4091-b7c7-10a169635a9a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.217392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-f562c8af-67, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:38.221375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:38.221887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:38.222384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:38.222989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-f562c8af-67, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f562c8af-67d2-4171-8236-1336b7353fdf', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:56:46:6a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:38.277220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[461953c8-8cb8-49cc-b895-40f449b045e0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143256, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.290736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3685e1e6-a71e-4829-838c-a9a579a2120b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.308698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee704d06-e637-433c-a94d-fe2c6a677971]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.322518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7ee40c38-c674-45e2-bc9a-334b8dee815e]: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:38.322518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:38.322518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:38.322518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:38.322518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:38.322518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:38.322518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:38.322518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:38.322518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:38.322518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:38.322518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:38.322518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:38.322518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ee40c38-c674-45e2-bc9a-334b8dee815e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:38.324032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:38.326143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:38.329503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2beb9f19-ef2c-4076-a32b-9a213da8c068]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.329503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:38.329503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:38.329503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:38.329503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:38.330087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40995a78-1f89-4b0e-a112-6206ac5ff615]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.330743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80f9e34e-c209-465a-9717-5dba0f1b194c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.331871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cdacec74-7bee-47ba-9fbf-f0caa389b221]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.341877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38f36cf8-6e7b-4348-b11e-4b865e0c3b40]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.362273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0bceb930-fbf6-4060-806c-334a5892b5ad]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 309430, 'reachable_time': 15648, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143270, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.374785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:38.376282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8cc64e2-f893-49c8-93e8-064ca93a857b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:38.377101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-617448100-router", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98" Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:18Z", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:66:5d:8b", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2039, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:19Z", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:28Z", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f562c8af-67d2-4171-8236-1336b7353fdf", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0cb463ae-0884-401c-b0c4-7519eb64a440", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:56:46:6a", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2040, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:21Z", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:24Z", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:38.380053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id fa412ee9-326b-499a-bc1e-2d694676db05. Wait time elapsed: 15.636 Jul 01 09:36:38.399984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id fa412ee9-326b-499a-bc1e-2d694676db05. Wait time elapsed: 15.636 Jul 01 09:36:38.399984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:38.399984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[298c2ae4-2b86-4fd4-92a5-fe7aca24aec7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.412570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[822df8bf-8eb8-4e0d-a753-35e4d0d643bf]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.413125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:38.413396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:38.413660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:38.416014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:38.436780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:38.437147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad15b0d6-e347-4564-99f8-29d58dcb1a54]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.441302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f639a1d-80b7-4e71-ac17-09fe5f1e4f6c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.442265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:38.481270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:38.482286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:38.482927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:38.483794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:38.485158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:38.530661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[336e375b-ad6a-4a0e-a3a8-2d348286b94a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.568866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f77e1545-7684-46cc-84cf-b102b0822866]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:38.568866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:38.568866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:38.568866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:38.568866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:38.568866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:38.568866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:38.568866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:38.568866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:38.568866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:38.568866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:38.568866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:38.568866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:38.570146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f77e1545-7684-46cc-84cf-b102b0822866]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:38.570765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:38.573509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:38.576328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ab2d81c-e0d2-473e-ae8c-a99b18a8b139]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.576328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:38.576328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:38.576328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:38.577425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:38.578733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f1ab6bf-f0be-4523-b281-5b6702174ad8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.579940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69e42062-b555-4c3d-8647-7f47e67f9138]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.581019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5247b35-ae6e-4728-bbd1-05b6f273e7d9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.601804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29d59274-39fb-4138-9a55-d9b6ff1b7647]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 309450, 'reachable_time': 29236, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143285, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.606167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:38.606429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[937da213-d7e3-473a-acb3-29e9b3966197]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:38.607867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:34Z", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:38.612881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id c9266c6b-246e-4b1a-b825-2e8237beecb6. Wait time elapsed: 24.039 Jul 01 09:36:38.638952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id c9266c6b-246e-4b1a-b825-2e8237beecb6. Wait time elapsed: 24.040 Jul 01 09:36:38.638952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:39.567933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.957s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:39.568555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:39.569439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:39.569817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:39.570155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:39.572471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2da3c9d3-401d-4e18-990d-b68bdd7728a0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.587777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfaf996a-55f8-4b03-adc7-e8ec07f9db7b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.613856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[105be122-b6b7-41fb-829e-77f9c9d94e57]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.642355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffe9a302-db23-46bb-a0ca-31a8cfe588fd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143288, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.662889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76ffac56-bbf7-4728-a505-32b821c3a599]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.683267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b3b6705-c7f4-4f66-9d05-ec4220a81ec1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.705286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b087702-eda5-4521-8f5f-b125b20d25e3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.725843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee1c5c82-b540-4435-9404-ff26b2c19c23]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.726494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:39.726770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:39.727036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:39.727748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:39.745703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.311s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:39.746251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:39.747440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:39.748065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:39.748350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:39.749650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:39.758013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb5b203f-0260-4b9f-a79e-1946c9b4e72c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.758575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b04e6d05-b35d-400f-a334-acaebd2bcb49]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.764034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec47e9ad-2128-4507-9d4a-85e24c58f17f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.765576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:39.800631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60d24661-6c5a-443c-88b5-92a617ccc3d6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.808849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:39.810151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:39.810737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:39.811564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:39.812151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:39.847140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d5fc9af-0058-49d7-bee8-d6ed9a7843e2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.867942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74e9336e-02f8-4056-8148-d0bf8962ac4f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.910827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d4a51e9-e61f-49a5-b20d-60cb2dfe456c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143307, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.921252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7c124522-dcf0-4669-8380-4f9290854f31]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:39.921252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:39.921252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:39.921252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:39.921252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:39.921252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:39.921252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:39.921252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:39.921252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:39.921252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:39.921252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:39.921252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:39.921252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:39.923240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c124522-dcf0-4669-8380-4f9290854f31]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:39.924362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:39.926799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:39.931418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03322743-6924-438f-8a42-29302dea13dd]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.932408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:39.932820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:39.933130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:39.933833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:39.935768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08a8beb3-8bc0-4ee2-a26f-b82a082d82f9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.936833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af6c4e5b-c372-4303-acef-d7c581e1b505]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.938676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e70f549-7512-457b-a956-89baf6e0c285]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.939042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f05076ec-079b-42eb-8f1f-4c608388dc7b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.965908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e518d24-3800-499d-8e22-2814e12768ec]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.972308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0cace6c-7269-4590-bdf6-ad90abb0f2cf]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 309590, 'reachable_time': 21872, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143312, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.981936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e176a30-7876-4f7c-bcf8-8ef2becd6750]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.983799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:39.986033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[468e39d1-74c8-49dd-8563-e4a8ae49e9f5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:39.987843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:34Z", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:39.991735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:40.007308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 431852e6-8d33-429b-a28c-5c863668fe44. Wait time elapsed: 29.004 Jul 01 09:36:40.007308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 431852e6-8d33-429b-a28c-5c863668fe44. Wait time elapsed: 29.004 Jul 01 09:36:40.007308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:40.023085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9482f6d7-f96b-4491-aabe-f6bfc4e739f8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:40.023759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:40.024131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:40.024514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:40.024987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:40.041644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:40.042002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[760fb76f-6db7-460a-b8a6-74cd807e5d61]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:40.045114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[238a3b64-6558-45e9-b307-fa298dace0f0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:40.047325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:40.082723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:40.083426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:40.083893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:40.084504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:40.085073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:40.129655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05764b9b-4c02-4d74-8dce-3a3d30c60bc9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:40.160892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cbb660cd-ee12-4b67-bc80-18dcd2d3a6d7]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:40.160892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:40.160892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:40.160892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:40.160892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:40.160892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:40.160892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:40.160892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:40.160892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:40.160892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:40.160892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:40.160892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:40.160892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:40.162308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbb660cd-ee12-4b67-bc80-18dcd2d3a6d7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:40.162402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:40.164112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:40.167648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75e4d907-cd9f-486b-aef7-a5b8f255ba37]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:40.168469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:40.168873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:40.169133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:40.169541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:40.170611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd71d705-ed12-4861-a943-82cc4bbd77b8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:40.171919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58ff92a2-4d0e-4d69-852f-8270862b522d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:40.173219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53f4650a-81e2-4e96-99e7-11a1eb792b5a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:40.200231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48fa5588-f1d5-48d7-8c78-bd410fc58d64]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 309617, 'reachable_time': 27964, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143328, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:40.205850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:40.206050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31775320-e3c6-4fe9-acb8-dd3de71e941b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:40.207105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:33Z", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:40.211540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id 01635dd2-a624-4888-bb4b-9c19d82d5c33. Wait time elapsed: 25.510 Jul 01 09:36:40.230372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id 01635dd2-a624-4888-bb4b-9c19d82d5c33. Wait time elapsed: 25.510 Jul 01 09:36:40.230372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:41.138670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.930s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:41.139494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:41.140463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:41.140796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:41.141080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:41.142442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f46d422d-1b09-47f6-b7b4-a14582d62c2a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.155243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7973b850-34c2-4337-8540-8d8f4c291390]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.176602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[117d1b69-7712-4cec-a94d-94bb19e18e1e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.205733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc21a83a-19e6-44f6-9f8d-6f1dac7f0b11]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143334, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.225969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[542a2062-69db-4abd-96fb-5091856a0450]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.246017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca381893-a527-4d49-befe-cb1848e9a7cd]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.268109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7350148-249d-426e-9286-55cf8bbae02e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.290350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70697221-b497-40d8-bdb7-5f882ba5cbeb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.290861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:41.291178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:41.292088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:41.292088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:41.308257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:41.308555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3432da7-a176-409b-9706-ba3298df1ce2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.311075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9bc13dd-2947-4d17-9a80-f57857ef3d90]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.312842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:41.346834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:41.347821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:41.348428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:41.349042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:41.349704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:41.378522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d04c7790-99c0-4e5e-a87a-04d60bdc8c6c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.433109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e1463cba-8fc4-49de-967e-a8997a2bcb72]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:41.433109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:41.433109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:41.433109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:41.433109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:41.433109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:41.433109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:41.433109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:41.433109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:41.433109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:41.433109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:41.433109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:41.433109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:41.436166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1463cba-8fc4-49de-967e-a8997a2bcb72]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:41.437074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:41.439058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:41.443942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbe8ae40-ae69-4f94-8a0a-607c05fd913e]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.443942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:41.443942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:41.445467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:41.445808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:41.446914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c703f4d6-29fc-4cd1-a361-818cc3c8d512]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.448181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e13d6270-2157-4f47-bb5a-9c3f8ebda09b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.449394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4aa62b5-268d-4f29-9b1e-8301cfd2e26e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.469215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b15e159-70b1-4d7c-8cd9-31fc6fd46e8b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 309746, 'reachable_time': 27140, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143356, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.472242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:41.472742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[077e4b80-a228-4bd2-b106-6d770b3bae94]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:41.474523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:40Z", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:41.479255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:41.502792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id c9419d93-e4b7-4bc6-b96e-6b06ff675fa5. Wait time elapsed: 24.412 Jul 01 09:36:41.502792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id c9419d93-e4b7-4bc6-b96e-6b06ff675fa5. Wait time elapsed: 24.413 Jul 01 09:36:41.502792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:43.118593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.738s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:43.118593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router b361ccc3-7577-4438-b123-f16e00901596 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:43.118593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:43.118593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:43.119731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:43.120104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1d410fd-e571-4316-baef-273a6ebd9e32]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.155784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73bc57f5-5f7f-4998-ac57-c73edcccbd61]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.187418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d6e0f5e-212e-4e22-bdec-cef74052a317]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.246286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74a56640-a64c-458c-aa21-f7679a5752de]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143380, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.289542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1409f07e-081f-4d37-a138-5f7bce915661]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.351331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f70ee91-0362-44ff-835f-1d8eb37405b3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.394406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[756f9217-701f-4e0c-857d-cab2f53cd20b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.463491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27ddcefc-c6da-4cd6-bbc0-fc36e86d1aa5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.464287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:43.464746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:43.465190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:43.465639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(f562c8af-67d2-4171-8236-1336b7353fdf) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:43.548626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:43.549176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7fa93d9d-a79a-4b4b-85d2-d6597269a01e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.552232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[087d683b-0b19-458c-addc-52f7561b687a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.553418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-f562c8af-67, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:43.597143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-f562c8af-67, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:43.600283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:43.600944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:43.601548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:43.602634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-f562c8af-67, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f562c8af-67d2-4171-8236-1336b7353fdf', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:56:46:6a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:43.653461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1eddb67-f14d-49b5-b7b4-8c81f42f8b84]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.687402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c7bae0b1-12e1-40fd-99fb-2f5ab23a552f]: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:43.687402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:43.687402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:43.687402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:43.687402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:43.687402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:43.687402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:43.687402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:43.687402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:43.687402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:43.687402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:43.687402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:43.687402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7bae0b1-12e1-40fd-99fb-2f5ab23a552f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:43.688802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:43.691595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:43.694397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89de15c2-1dac-4fc2-aedc-e7bd5d782366]: (4, ['qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.694397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:43.694397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:43.694397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:43.697337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:43.698221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11ced66c-7ae0-4d03-8007-8f05a0778907]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.700097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5da93d36-f3c0-4055-bfcc-8729017ad0de]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.703499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac8b727f-32d1-4913-b61f-31b9d7774f9f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.776745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3771bff-3fd9-485e-865b-b97d6a4837dc]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 309950, 'reachable_time': 43796, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143441, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.784802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:43.785289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[527a05a6-1a69-4ad8-ba62-76a836ca9ab0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:43.786489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-617448100-router", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98" Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:18Z", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:66:5d:8b", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2039, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:19Z", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:28Z", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f562c8af-67d2-4171-8236-1336b7353fdf", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0cb463ae-0884-401c-b0c4-7519eb64a440", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:56:46:6a", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2040, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:21Z", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:37Z", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:43.790760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:43.816725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id b2894539-fbf2-4e7c-b8a0-7d742342f0cb. Wait time elapsed: 22.883 Jul 01 09:36:43.816725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id b2894539-fbf2-4e7c-b8a0-7d742342f0cb. Wait time elapsed: 22.883 Jul 01 09:36:43.816725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:44.147265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-427c27bf-741d-49af-a0de-db4dd98e574d tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['203e4741-69b9-468f-97b4-76a7d63b11b8'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:36:44.828636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.037s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:44.828901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router b361ccc3-7577-4438-b123-f16e00901596 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:44.829876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:44.831117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:44.831822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:44.834891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e093745b-fb6c-4f5b-81c0-7dd0e39fee7a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:44.861236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6df023ab-5758-4b97-b080-8ff2573686dd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:44.886553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ba7d9b7-aa04-49e7-9390-cc3301d91769]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:44.925383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94c82dcc-dbf2-4e9b-a0fc-3c2b7592ac96]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143485, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:44.947270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0732e85d-0ab0-46fe-9cdb-0d074bb63fca]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:44.970420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1cd2ea0b-fb11-4a5f-9e50-bdaf70802d15]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:44.995013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[960d72b1-2a90-4747-adad-5b3649b1d7d5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.017801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6049223-13a4-4733-9de2-1a3948a56571]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.018651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:45.019081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:45.020475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:45.021008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(f562c8af-67d2-4171-8236-1336b7353fdf) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:45.039389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.047s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:45.039870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:45.041429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:45.042029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:45.042467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:45.045087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38420e3f-8f1b-4088-b696-03c0dd37591c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.051729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:45.052352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93707592-162d-4d51-8acf-0a14996c8fb5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.055139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e1bb248-29b8-477c-9be5-a583c9b03d4b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.056667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-f562c8af-67, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:45.076702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36128f43-8eec-4753-aafe-192027a579d3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.093923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-f562c8af-67, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:45.095446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:45.096002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:45.096927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:45.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-f562c8af-67, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f562c8af-67d2-4171-8236-1336b7353fdf', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:56:46:6a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:45.141257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5fa8b1c-7a64-4910-8f41-56681e0164e8]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.158927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90227459-7be5-455c-9dc7-dd639f85988b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.202529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23812560-2810-40c8-b1b8-9a17337fc2ac]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143505, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.207898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a198cb07-fc59-428f-a22d-ebd2aec3b56c]: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:45.207898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:45.207898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:45.207898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:45.207898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:45.207898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:45.207898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:45.207898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:45.207898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:45.207898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:45.207898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:45.207898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:45.207898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:45.209528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a198cb07-fc59-428f-a22d-ebd2aec3b56c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:45.210443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:45.212237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:45.216442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[191b82a2-7fdc-47d9-a272-f8865b60858f]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.216442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:45.216442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:45.216442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:45.216442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:45.219823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bf027f8-7110-42c8-b71c-0dfd2cff5131]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.222917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9b5a22d-c139-4130-83b9-8b45d85edd84]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.224382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2345ab5e-a5a8-40ff-a1dc-bc682f833373]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.249409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83d69151-00a0-4995-b871-0609c6341647]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.255689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e535f612-280e-4bf5-9468-a7c16a56cfec]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 310118, 'reachable_time': 31172, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143510, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.269379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:45.269871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94d45e0f-5bd4-420d-80f7-fb7737625e79]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:45.271287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-617448100-router", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98" Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:18Z", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:66:5d:8b", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2039, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:19Z", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:28Z", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f562c8af-67d2-4171-8236-1336b7353fdf", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0cb463ae-0884-401c-b0c4-7519eb64a440", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:56:46:6a", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2040, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:21Z", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:43Z", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:45.274815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id fa412ee9-326b-499a-bc1e-2d694676db05. Wait time elapsed: 22.530 Jul 01 09:36:45.286994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id fa412ee9-326b-499a-bc1e-2d694676db05. Wait time elapsed: 22.531 Jul 01 09:36:45.286994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:45.300872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20e17b49-8f24-4c73-8d32-08eba3abb597]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.323353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c1e443e-52ba-4c03-8ed8-fc11e9eefb05]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.364789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1bbdd813-0039-4116-98c4-869f3e29f05f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.365639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:45.366084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:45.367183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:45.367701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:45.391716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:45.392237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bac6e865-e7c7-4b15-a00a-879aac91788b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.398163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e9dbd9a-e6b2-463f-95d5-eebfb17a2bbd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.399161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:45.435974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:45.436376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:45.436768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:45.437203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:45.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:45.480707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23e34392-b270-4323-91c2-c708c624e17f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.517806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d74e6490-0340-4f43-b344-881b1be9a7ca]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:45.517806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:45.517806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:45.517806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:45.517806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:45.517806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:45.517806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:45.517806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:45.517806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:45.517806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:45.517806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:45.517806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:45.517806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:45.518953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d74e6490-0340-4f43-b344-881b1be9a7ca]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:45.519063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:45.521132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:45.524722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e21c331e-2687-428d-ac96-346f655fc510]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.524722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:45.524722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:45.524722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:45.525327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:45.526465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c95e3b0b-0cb5-48ef-8e42-6cf150d985fa]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.527547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[baa07354-f3ed-4ffe-a2b0-0a39493a83e9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.528586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2887c15-6b77-4d32-b910-1113efcfa8fc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.551835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7f1e28e-4f83-470e-94ff-3896b775dd36]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 310146, 'reachable_time': 24760, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143531, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.554702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:45.555197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa2274bc-58b4-4ad8-92f5-10a27d07ed77]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:45.557099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:42Z", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:45.560634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:45.583275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 1, update_id dd2204c3-586b-4fe2-83ed-a05211a04e6d. Wait time elapsed: 1.412 Jul 01 09:36:45.583275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 1, update_id dd2204c3-586b-4fe2-83ed-a05211a04e6d. Wait time elapsed: 1.413 Jul 01 09:36:45.583275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:46.478139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.916s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:46.480784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:46.481989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:46.482332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:46.482620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:46.484458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca656a9e-94af-43b0-a151-24540472e65c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.504595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[940726ff-9cd0-4777-bc8d-73399e108b26]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.524947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4be20a98-d324-4235-ad2a-2e69e2ac332e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.582705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06b9786c-5424-4ac6-9618-650a32de0e64]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143549, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.634347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dce63a92-805a-4a5e-ab0e-1d7bde2728ce]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.672962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed810884-784e-407d-bd30-a7c84652f931]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.705615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15a300df-99e2-4d56-b881-c623332c1c50]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.727346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ec1fdb4-421e-4040-bc48-358934f22fbe]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.728062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:46.728424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:46.728754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:46.729096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:46.747746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:46.748023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28206f0e-a776-493c-b79c-670d5b736afc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.752467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a888d2c-113f-43f4-b913-64d32714dccd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.753577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:46.784797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:46.785498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:46.785897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:46.786290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:46.786675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:46.840583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.360s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:46.840939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:46.843206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:46.844370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:46.845135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:46.846587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9a10d35-2d92-4742-ac7a-1ebc53a9677d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.850305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[634a05c9-14eb-4a40-b54f-7de95530c393]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.876005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53d0615f-2229-478e-b2a1-318bd6852521]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.898905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[533409a0-430c-44c1-821c-84ef947ae5d4]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:46.898905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:46.898905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:46.898905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:46.898905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:46.898905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:46.898905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:46.898905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:46.898905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:46.898905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:46.898905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:46.898905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:46.898905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[533409a0-430c-44c1-821c-84ef947ae5d4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:46.900291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:46.902569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[186fbb18-2796-4a18-a746-7dd1bb0a849e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.906207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cacd4c61-2bb7-41ba-933c-b7a54aa97a27]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.907584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:46.908479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:46.909047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:46.909730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:46.911193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed581e08-1090-4b75-b44e-307cb5648c81]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.912488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc1bd320-e1cf-4407-bb07-bfdd7747b029]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.914089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c06572c-3335-4b2b-853a-953952a3d40f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.945609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77938e96-9eab-4b66-baf9-5caa0e674a7b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 310284, 'reachable_time': 41920, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143588, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.954809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0371832-7fae-4a8b-ba0f-8aeea04bd100]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143587, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.956413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:46.957133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1670e7f1-9996-4b78-b77b-c9b8e734c6fe]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:46.958880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd0fbd76-f17e-48d7-a3c6-89b4557b3537", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.113", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "72e3e169-abbc-435b-a035-9cfe735aacbf", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.9", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2061, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:f0:18", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1912bdb1-a7cc-4930-a1f3-dbfcaf7fe0d7", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:42Z", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:42Z", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:46.963807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:42Z", Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id c9266c6b-246e-4b1a-b825-2e8237beecb6. Wait time elapsed: 32.394 Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id c9266c6b-246e-4b1a-b825-2e8237beecb6. Wait time elapsed: 32.395 Jul 01 09:36:46.979256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:46.982063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[864108c3-b473-4c0c-81d8-a9365b8fe1dc]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.007634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a13d0c15-9559-4523-bd1d-79f434b7a0ee]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.031057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbdcd662-e899-46e9-b754-bc029cfd2ed5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.056318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e31823b6-6f49-45ba-b4d0-f91caef11d32]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.057070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:47.057356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:47.057625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:47.058439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:47.076923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:47.078561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c332943-0126-4ad6-8108-1a3f9a19cd68]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.081938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[424fa648-9a72-4c03-b73d-1c0b6638cd40]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.083387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:47.142851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:47.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:47.145296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:47.145959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:47.146684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:47.191343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de33f580-baa3-4e99-a2b2-f78d1a49ecd5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.232849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3a6c77b0-a0d5-490a-9bc0-6dc18b0351ac]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:47.232849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:47.232849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:47.232849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:47.232849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:47.232849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:47.232849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:47.232849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:47.232849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:47.232849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:47.232849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:47.232849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:47.232849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:47.234627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a6c77b0-a0d5-490a-9bc0-6dc18b0351ac]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:47.235841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:47.238086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:47.243717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9efa20c7-bc4a-47a7-8c9e-92aa77f8548c]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.244865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:47.245266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:47.245532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:47.246025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:47.247240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2e03a47-335f-48c5-ba80-fb47f5ffdee5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.250307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a29ae091-b806-49e9-ad38-07b248eba156]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.252070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02742726-fe9a-4a01-a500-3abe6dce8e82]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.279998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54f481e7-a977-45d4-8a2e-c8e41cc48d2f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 310321, 'reachable_time': 18556, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143606, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.299233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:47.299892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71ab7350-55c5-4424-a0e2-8f238e8634a5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:47.301218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:40Z", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:47.305910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:47.320998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id 01635dd2-a624-4888-bb4b-9c19d82d5c33. Wait time elapsed: 32.609 Jul 01 09:36:47.320998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id 01635dd2-a624-4888-bb4b-9c19d82d5c33. Wait time elapsed: 32.615 Jul 01 09:36:47.320998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:48.070799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.755s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:48.072381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:48.073337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:48.073670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:48.073957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:48.075146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bab06dd1-fada-4c18-a358-fa4c224f1460]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.090416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db68e30e-5830-4954-a701-277bf4266d89]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.115097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a77ce756-eb20-48e4-90d9-c90c87b4d3ec]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.144884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce2a28a6-28b0-4600-a3a9-42d07a28df2d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143623, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.169175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10706084-fef6-4e35-9115-f5612336abdd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.193989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92e7b7ec-3c83-4eee-8f26-5d722f835485]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.216265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c66048ea-1cad-4e24-803d-8dda01daae93]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.239746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2234adc3-1355-48c0-97f1-5c2012b08ba1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.240482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:48.240865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:48.241215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:48.241562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:48.258937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:48.259785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aec0e43e-54bb-4141-96fa-679c5c9190d8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.262368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43a0d81e-3460-4e78-813f-5b7486c26ebe]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.263337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:48.297814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:48.298808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:48.299211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:48.299637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:48.300030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:48.339933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a6f5a87-08ed-4b3a-a3e7-f0a8fc101ef9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.387452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6ac2afc4-f417-4e39-958c-25a746ec50f7]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:48.387452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:48.387452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:48.387452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:48.387452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:48.387452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:48.387452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:48.387452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:48.387452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:48.387452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:48.387452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:48.387452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:48.387452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:48.388510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ac2afc4-f417-4e39-958c-25a746ec50f7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:48.389174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:48.391561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:48.393818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3417dee-7e11-4b24-ba05-aeb7527788ae]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.393818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:48.394025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:48.394108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:48.394440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:48.396421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d96635b4-a962-465f-99e8-4364a80bd74d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.397650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50a35de7-67f1-4156-bf15-811f8706e708]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.398890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4187d655-22a2-4941-98d6-334d393b632a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.420044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2ee0b66-b80a-42be-97ad-a67b3952ab16]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 310440, 'reachable_time': 42232, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143642, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.427111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a0106d4-478b-43a9-8c97-8df5411b803b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:48.429301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:45Z", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:48.432380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3 Jul 01 09:36:48.447575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id c9419d93-e4b7-4bc6-b96e-6b06ff675fa5. Wait time elapsed: 31.363 Jul 01 09:36:48.447575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id c9419d93-e4b7-4bc6-b96e-6b06ff675fa5. Wait time elapsed: 31.363 Jul 01 09:36:48.447575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:49.183723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.909s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:49.184364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router b361ccc3-7577-4438-b123-f16e00901596 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:49.184898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:49.185217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:49.185492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:49.186782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83488797-e8aa-4c88-9f94-22524ea295d6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.206559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4332cb2f-5d82-4695-9870-a50b5c460009]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.231276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fedfbfb2-17c6-464a-8640-75815ace0ecb]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.268169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[adca04f6-076d-4df4-9ccd-6300f6225650]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143646, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.290381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da5608e7-28bf-486b-8f91-67e70e941dad]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.311941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6646d46-ae4e-40f4-832b-9586b201d6eb]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.335020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57a0df6a-6b55-4b57-a5d6-c4fd08359369]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.357372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[059c6b21-21d8-46f3-a0ee-32122a3c6f74]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.358198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:49.358493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:49.358743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:49.359046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(f562c8af-67d2-4171-8236-1336b7353fdf) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:49.379841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:49.380269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3c813af-aec2-421e-8e28-8ad04ce9f58f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.383200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff9273ae-6734-43ab-bb3d-a5731e2168ab]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.385166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-f562c8af-67, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:49.417510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-f562c8af-67, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:49.418675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:49.419303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:49.419843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:49.420327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-f562c8af-67, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f562c8af-67d2-4171-8236-1336b7353fdf', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:56:46:6a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:49.460527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25ca12b1-bfa8-4cae-8af5-b8741c794a2e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.517546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[09f9da43-436c-43f2-91a4-ea8be9ac535b]: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:49.517546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:49.517546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:49.517546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:49.517546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:49.517546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:49.517546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:49.517546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:49.517546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:49.517546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:49.517546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:49.517546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:49.517546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:49.518533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09f9da43-436c-43f2-91a4-ea8be9ac535b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:49.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:49.522058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:49.524502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff108337-c3c9-4bb1-b63b-1caa2597d6d6]: (4, ['qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.524502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:49.527250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:49.527444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:49.527977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:49.531782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ccb8579-81b5-4b94-a377-b5901f51a024]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.532807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6326df9-8a04-4df0-ab67-6f7fa81f2526]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.534642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e41cf48d-686e-4200-b831-dd5179f74330]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.562671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bae42ff0-c911-4bff-808f-413042c498c9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 310553, 'reachable_time': 18052, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143664, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.572919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:49.573488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[adeff2ce-c8d6-437c-a1a5-2c2e8a320aa3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:49.574750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-617448100-router", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98" Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:18Z", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:66:5d:8b", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2039, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:19Z", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:28Z", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f562c8af-67d2-4171-8236-1336b7353fdf", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0cb463ae-0884-401c-b0c4-7519eb64a440", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:56:46:6a", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2040, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:21Z", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:43Z", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:49.580605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:49.602231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id b2894539-fbf2-4e7c-b8a0-7d742342f0cb. Wait time elapsed: 28.672 Jul 01 09:36:49.602231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id b2894539-fbf2-4e7c-b8a0-7d742342f0cb. Wait time elapsed: 28.673 Jul 01 09:36:49.602231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:50.305581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.339s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:50.306218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:50.307786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:50.308360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:50.309019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:50.311745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9d34a54-08c7-4c74-9fa3-0af52e3d49ee]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.331680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f56f5293-7d85-43fa-8081-b670ab86e036]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.361693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ceac385b-5bb9-4c29-84d9-8327cdcd4b0e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.392663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49d8a7bc-a7e9-4e6f-bee0-5fce849b6311]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143682, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.413745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7983224-f83c-40da-9db7-794281fe2bc3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.451014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a375af0a-9d57-4bdb-8b8d-f05dc329bbaa]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.473777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a1d7601-e901-479b-baec-1e767476a7ab]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.494976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[637ff7aa-c737-4ab3-b921-43888d1d4258]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.495627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:50.495908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:50.496176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:50.496458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:50.503941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.922s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:50.504266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router b361ccc3-7577-4438-b123-f16e00901596 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:50.505296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:50.505804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:50.506147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:50.510961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4133982e-5382-45ea-a896-b5d059e19a70]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.515279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:50.516157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ae0405a-5d5a-4efa-b74e-6661f738eb47]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.519328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[866c477c-e608-4705-b610-390c44113c76]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.520578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:50.536263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2182dbd-3b30-43f5-8af4-173bd8b19d72]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.564892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4edec5e4-e36c-4d18-b011-bc535c667925]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.567048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:50.569857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:50.570261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:50.570657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:50.571046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:50.607152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf012e96-5261-48be-9bdf-a9112ec1b165]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.668513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bf41d87-7c21-428f-a178-a11a6c11a56b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143693, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.679274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[317d0d19-b1d3-455e-ac3b-502a2bba5047]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:50.679274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:50.679274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:50.679274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:50.679274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:50.679274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:50.679274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:50.679274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:50.679274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:50.679274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:50.679274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:50.679274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:50.679274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:50.680401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[317d0d19-b1d3-455e-ac3b-502a2bba5047]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:50.680487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:50.682281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:50.684521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f745bfeb-be4f-46fc-8784-cc43c2e026af]: (4, ['qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.684521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:50.684521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:50.684521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:50.684521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:50.688126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90d597e8-a3b3-48c3-9319-881a90cbbf1f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.689230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a31d5547-1d60-4e33-9c0e-2c89d76f6476]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.690296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02fc31ef-33c4-4d5e-900d-fd8451319587]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.711349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d14cfed-5c93-4b5f-864e-d5f1ec19b2f6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.745091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d04b099-874f-4bd8-b1c5-630310133977]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 310665, 'reachable_time': 19796, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143705, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.748949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:50.750586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fdf1f3f-e65e-4ed1-b9ab-79511dfa795c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:50.752091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd0fbd76-f17e-48d7-a3c6-89b4557b3537", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.113", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "72e3e169-abbc-435b-a035-9cfe735aacbf", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.9", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2061, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:f0:18", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1912bdb1-a7cc-4930-a1f3-dbfcaf7fe0d7", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:42Z", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:42Z", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:50.756556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:46Z", Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 431852e6-8d33-429b-a28c-5c863668fe44. Wait time elapsed: 39.768 Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 431852e6-8d33-429b-a28c-5c863668fe44. Wait time elapsed: 39.769 Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:50.776889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ef09fe2-5545-4856-8b6c-78fc4099a052]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.805307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[172023e3-c3a7-4f90-a173-bb942af97cc5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.843841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[878281ff-5a0e-4882-9ce9-05108fda7098]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.844740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:50.845112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:50.845606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:50.846034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(f562c8af-67d2-4171-8236-1336b7353fdf) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:50.870661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:50.871209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e369a84-c0ce-4d83-aa86-794b70820d9b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.876290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c446593-af05-48ce-ba8b-0478016e4ac9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.878766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-f562c8af-67, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:50.921231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-f562c8af-67, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:50.923203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:50.923676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:50.924067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:50.924451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-f562c8af-67, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f562c8af-67d2-4171-8236-1336b7353fdf', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:56:46:6a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:50.963769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.533s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:50.964399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:50.965570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:50.966129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:50.966902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:50.967966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89a96ab0-ad80-44e1-9593-065d2a7617f2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.969828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b78c88f-95df-4e0a-a5dc-16f3ca7ec4cf]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:50.990082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8170b1a1-ab1e-4ed8-96ce-c339b6e48f78]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.013691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d65d13ec-125f-43ca-a063-d34e6717cbed]: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:51.013691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:51.013691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:51.013691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:51.013691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:51.013691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:51.013691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:51.013691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:51.013691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:51.013691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:51.013691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:51.013691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:51.013691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:51.015218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d65d13ec-125f-43ca-a063-d34e6717cbed]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:51.016587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:51.018661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ccc83441-638a-433f-b051-664f7ab4e0ac]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.021931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e14053a4-3e16-4d4f-a5ca-7e4b2c7a5a92]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.021931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:51.021931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:51.021931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:51.022296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:51.023070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ad02181-b101-49d1-b2de-e95fbb0c2378]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.024188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5fe9e48-531c-4193-ba09-15cdea1a21ef]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.025171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51f124f4-e0e4-4ea8-b887-1910555423a1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.048423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6aaec2ec-eca1-4d4b-ae37-9d12d57e7feb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 310692, 'reachable_time': 42440, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143724, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28f25ce6-e1e5-45ce-93e9-d5530bf1ee85]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-617448100-router", Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98" Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:51.056269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:18Z", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:66:5d:8b", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2039, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:19Z", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:28Z", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f562c8af-67d2-4171-8236-1336b7353fdf", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0cb463ae-0884-401c-b0c4-7519eb64a440", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:56:46:6a", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2040, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:21Z", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:49Z", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for b361ccc3-7577-4438-b123-f16e00901596, action 3 Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id fa412ee9-326b-499a-bc1e-2d694676db05. Wait time elapsed: 28.310 Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 2, update_id fa412ee9-326b-499a-bc1e-2d694676db05. Wait time elapsed: 28.311 Jul 01 09:36:51.063902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:51.091824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0125f56e-7ea1-49ad-a3f5-a3b9aa09d8d5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143723, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.113346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d55c60e7-ff8e-4798-bef8-dc28b98752fb]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.139955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77f94ab9-7531-4cee-b34f-4a3a42174b5b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.181121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[585837bb-8648-4d4f-8974-f1ca9b7a4cd7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.225373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb1fb331-6d05-4eaa-b65a-6ac6b6498d4d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.226321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:51.227749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:51.228270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:51.228773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:51.259816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:51.260473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7708a151-f799-4e46-b8ac-72db02c97634]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.263585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31c150ee-908f-4f94-8f06-adf156eab202]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.265715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:51.301659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:51.302784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:51.303511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:51.304132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:51.304750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:51.352124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c61e3db7-a54d-4ede-9a3e-9dcbba69bcc3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6f007e15-8929-410b-a471-17885462806f]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:51.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:51.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:51.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:51.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:51.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:51.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:51.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:51.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:51.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:51.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:51.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:51.403393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:51.406632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f007e15-8929-410b-a471-17885462806f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:51.407730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:51.409518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:51.411800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b216ea7-7efa-4c8e-8d78-07126beb4ddb]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.411800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:51.411800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:51.411800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:51.411800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:51.413088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ccb7fa3b-f93b-467e-8383-d1dca796e6be]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.414060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e05d1d4-f7d5-42b0-80dd-5660faf716b6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.421559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b47bb07-e09a-45fb-8857-b28fdfa7ce38]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.458193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b86c13a0-84d5-42f9-b784-eb5c01b1687e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 310734, 'reachable_time': 29204, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143742, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.469616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:51.470841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecc98cd9-550f-477a-bb0f-2cda75bfa9bc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:36:51.473934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:45Z", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:51.476967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3 Jul 01 09:36:52.061397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.006s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:52.061673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router b361ccc3-7577-4438-b123-f16e00901596 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:52.064038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:52.064434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:52.064784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:52.066579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[482612d3-6849-4f1f-87e8-2532fe0e6b07]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.099038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06329ff3-7e4d-4b02-8a66-9a4c4ce56c87]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.127069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3cc0210b-8e07-4d02-b7d6-913997dd1b21]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.168360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8acb511a-107b-479c-b6cd-886dcb387053]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143749, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.196855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e333f41b-6b19-4711-8319-5a837c606905]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.229713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e45aa35-856e-477f-8bf1-bd86810afdef]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.252997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4eba920f-ad71-4f51-b474-d4b2fe198d0c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.277288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4e7ee1d-a5d3-44c9-883c-70083be0467a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.278109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:52.278405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:52.278662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:52.278940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(f562c8af-67d2-4171-8236-1336b7353fdf) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:52.321403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:52.321659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f0433f8-cdae-4283-bd12-3dcf481e4319]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.325116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1680d450-32e2-48bf-b6f5-e87347f55854]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.326503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-f562c8af-67, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:52.367833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-f562c8af-67, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:52.368476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:52.368871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:52.369371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-f562c8af-67, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:52.369635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-f562c8af-67, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f562c8af-67d2-4171-8236-1336b7353fdf', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:56:46:6a'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:52.409884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c878a9c-0b97-4e9b-9ac8-4b139a3dd4da]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.456623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b76ea81b-740c-4005-9776-63a26b2fa195]: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:52.456623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:52.456623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:52.456623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:52.456623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:52.456623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:52.456623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:52.456623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:52.456623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:52.456623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:52.456623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:52.456623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:52.456623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:52.459739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b76ea81b-740c-4005-9776-63a26b2fa195]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:52.461106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:52.463048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:52.465347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a96e76e6-846d-41b7-b163-76600a5706f0]: (4, ['qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.465347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:52.465347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:52.465347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:52.465347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:52.468940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61b3f283-fa1f-4fd7-8d1b-ef90a5f54c14]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.471332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6645b8bb-30c1-4d16-929f-8fd717f7c00f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.472345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1f9212b-f033-42ff-9f19-3a2e679ae1e4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.504365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a14cc13-f4d9-4cb1-8f54-58f517b76b62]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 310843, 'reachable_time': 24588, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143769, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.511075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:52.511715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1bae76d2-2c81-4d5c-9ad7-00df4d660683]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: b361ccc3-7577-4438-b123-f16e00901596: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-f562c8af-67 not found in namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596. Jul 01 09:36:52.512462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-617448100-router", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98" Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:18Z", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:21Z", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7e3649ba-21e8-45b7-815d-c38c69b13308", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:66:5d:8b", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2039, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.98", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:19Z", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:28Z", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f562c8af-67d2-4171-8236-1336b7353fdf", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0cb463ae-0884-401c-b0c4-7519eb64a440", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:56:46:6a", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b361ccc3-7577-4438-b123-f16e00901596", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2040, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:21Z", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:49Z", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "129ed948a2834d248f7fa6b1a02e67af", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b6c02109-44d2-41da-8123-37b5115275b0", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:52.516467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for b361ccc3-7577-4438-b123-f16e00901596, action 3 Jul 01 09:36:53.003382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.242s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:53.004541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:53.005624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:53.006125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:53.006574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:53.008605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5493532f-cc54-48e1-bc8e-48e274b9ed7d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.035963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[943e2a1d-3158-41fb-9af7-dccaba6106d8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.075820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58fd3ced-4f86-474d-bbe7-e859f335f86b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.114896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27cb56b8-9058-423b-ad6f-a9b522804b1a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143775, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.142549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbcdfb51-2e4b-4dc4-a20c-bdaf29b0b17e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.166229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[235dd6c5-b9b2-4a00-a19a-bc4a21c329ed]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.194151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1788113-8ce3-4043-ab72-68af467a2ee2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.221038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36a41abb-c4c9-45a8-8931-d76ce0192bb4]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.221943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:53.222568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:53.223068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:53.223723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:53.246777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:53.247576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27ad98bc-219a-478a-930b-7f2e761580a6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.252346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7eacf026-71df-4bcd-ab76-99b2b006cddd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.254252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:53.293340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:53.295340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:53.295928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:53.296782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:53.297366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:53.358059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc02d138-754d-49ec-bda1-e94d76bf7f08]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.416779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f0898bee-a9a2-485b-b7ee-3359cd5187e7]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:53.416779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:53.416779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:53.416779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:53.416779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:53.416779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:53.416779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:53.416779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:53.416779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:53.416779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:53.416779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:53.416779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:53.416779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:53.419302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0898bee-a9a2-485b-b7ee-3359cd5187e7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:53.420557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:53.423530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:53.427807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b245f4a-3738-4da8-89a0-6188f418f5e4]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.428869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:53.429636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:53.429903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:53.430250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:53.433016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c11f69eb-18a1-43b4-8726-2e289fc0652a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.434230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6833fdf-8f49-41b6-bae3-a953cf6f5190]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.435631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea91d7ec-e8ba-45f6-a05f-a35b5d6a08bd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.469052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eed35dd9-ed6e-456d-8eeb-f01729dfc339]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 310937, 'reachable_time': 19764, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143799, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.471084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:53.471391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acb29185-0ae1-4f8d-855a-8dce8c7de11b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:53.472463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd0fbd76-f17e-48d7-a3c6-89b4557b3537", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.113", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "72e3e169-abbc-435b-a035-9cfe735aacbf", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.9", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2061, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:f0:18", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1912bdb1-a7cc-4930-a1f3-dbfcaf7fe0d7", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:42Z", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:42Z", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:53.476600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:46Z", Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3 Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id dd2204c3-586b-4fe2-83ed-a05211a04e6d. Wait time elapsed: 9.325 Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id dd2204c3-586b-4fe2-83ed-a05211a04e6d. Wait time elapsed: 9.326 Jul 01 09:36:53.492786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:55.121358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:36:57.637898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.163s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:57.638496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:57.639132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:57.639482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:57.639764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:57.641179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be1841b4-43f9-40d6-9e45-cb2de4b11853]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.660064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2dd5d6b-fd90-4625-891a-11b38a95b3b1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.682277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3318d51b-e885-424b-86dd-184c8a9b4633]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.711517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36dcd9e0-49a0-4ccc-85c1-6d84e1b93fe7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143888, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.723198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9580b9f9-f3c6-4235-bc68-6414abc58dff]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.743961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ab37180-ed4d-4241-bf2f-bd33c4a038f8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.766902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0750db68-635d-4581-96ff-7863c5848768]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.787839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[105faccb-6e30-4787-9faa-5449a9de6a7d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.788614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:57.788974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:57.789302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:57.789641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:57.805382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:57.806068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2123a6c4-1b22-4975-94d1-40fd29b3770f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.808945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b60f9e24-d1ef-48a4-ad1a-c8d998586074]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.810025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:57.847325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:57.848367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:57.848936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:57.849502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:57.850086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:57.901505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[283e3bbd-74b2-40e8-b846-39fb0161b7f8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.938984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e385df62-e7f1-4316-892a-4a44773ff8b7]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:57.938984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:57.938984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:57.938984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:57.938984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:57.938984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:57.938984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:57.938984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:57.938984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:57.938984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:57.938984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:57.938984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:57.938984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:57.940644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e385df62-e7f1-4316-892a-4a44773ff8b7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:57.941994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:57.944059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:57.946308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29e7e49f-2778-4de9-8829-3678be714b4d]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.946308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:57.946308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:57.946308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:57.946651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:57.947701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a95699e2-8e37-42a8-9488-ad2f90ab3426]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.949315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2a425b5-fdc5-4610-b203-185391534f13]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.950710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9bc5f71-5862-4f13-bc09-bac81895b8bd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.974819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[557084f3-27d4-4c4b-93d9-017f69f933f3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 311397, 'reachable_time': 27668, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143909, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.977182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:57.978018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3cf0a160-8049-42aa-a0d2-79d0f3c1adf1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:57.979232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd0fbd76-f17e-48d7-a3c6-89b4557b3537", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.113", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "72e3e169-abbc-435b-a035-9cfe735aacbf", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.9", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2061, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:f0:18", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1912bdb1-a7cc-4930-a1f3-dbfcaf7fe0d7", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:42Z", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:42Z", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:57.988175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:54Z", Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id c9266c6b-246e-4b1a-b825-2e8237beecb6. Wait time elapsed: 43.428 Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id c9266c6b-246e-4b1a-b825-2e8237beecb6. Wait time elapsed: 43.429 Jul 01 09:36:58.012398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:36:59.096669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.097s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:36:59.096669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:36:59.097604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:36:59.097989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:59.098793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:59.099942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8daa83d9-e0bc-4eac-aee8-6143b8b0f8a7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.118152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8379223-fd3c-4b4c-8ec1-bc5224b0c2a9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.144356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86995656-ac58-4b06-a229-65dd5315368b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.179525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fcc2bea-01c9-4455-9a35-ef6e0711d0ee]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143918, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.201271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd72c8aa-8bcf-49eb-926d-b836a8d221c3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.221729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3f26296-0ea6-4483-adee-a6764fc259fa]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.241827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03b15d0d-0018-4b64-a641-c42f050123b8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.261948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[280ccc88-562f-47bc-afd3-6fde30501ae8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.262563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:36:59.262839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:36:59.263105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:36:59.263420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:36:59.281332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:36:59.281614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d66f37b-eb21-4a94-ad77-211443b9ca1d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.284301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[110a7ae6-ee8f-4f13-8843-779cd05cd8db]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.285227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:59.340183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:59.350137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:59.350679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:59.351288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:59.351785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:36:59.395968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a69ff577-d8d7-44a9-ac01-7d3ebc3eab0a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.431034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[448cd9ce-0119-4579-8f32-ea17b0829af4]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:36:59.431034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:36:59.431034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:36:59.431034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:36:59.431034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:36:59.431034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:36:59.431034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:36:59.431034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:36:59.431034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:36:59.431034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:36:59.431034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:36:59.431034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:36:59.431034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[448cd9ce-0119-4579-8f32-ea17b0829af4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:59.433395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95b98185-c719-4fee-822e-728d31e39cf1]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:59.438458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:36:59.439789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:36:59.440747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7a07bb9-deb1-439d-885a-bfa8e9ecc9c0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.442407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0484767-4008-4c04-b522-e0522c6ad9ac]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.443612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1b87dd0-3059-4fc0-b508-ff79abea7c9b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.471009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca0bec49-64cd-454c-945e-c21d285d340c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 311544, 'reachable_time': 23788, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143937, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.482717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:36:59.482974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc2b0756-64d6-45b3-bd08-92650a6c8682]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:36:59.484621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd0fbd76-f17e-48d7-a3c6-89b4557b3537", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.113", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "72e3e169-abbc-435b-a035-9cfe735aacbf", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.9", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2061, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:f0:18", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1912bdb1-a7cc-4930-a1f3-dbfcaf7fe0d7", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:42Z", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:42Z", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:36:59.488392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:54Z", Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3 Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id dd2204c3-586b-4fe2-83ed-a05211a04e6d. Wait time elapsed: 15.340 Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id dd2204c3-586b-4fe2-83ed-a05211a04e6d. Wait time elapsed: 15.340 Jul 01 09:36:59.502941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:37:01.326076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.837s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:37:01.326732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:37:01.328011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:37:01.328332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:01.328612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:01.329749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc8d4306-1b4e-4ada-aa4d-75a4fdae1397]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.340365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36ebed92-3d39-429c-b3bb-a9bdde8c2e24]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.365863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce0f36d2-160f-4c2a-b214-038c5706dec9]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.395492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89971edc-ff05-4b01-9906-ad6f2a526507]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143957, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.422437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3bd790e-ce4c-4a7a-8dbf-32bc60ed32b1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.442483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d26f2202-4dc4-4002-ac01-cc9ec44cb51a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.462888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b14f0e2-3fab-456c-88b2-8a1bcf8c6b94]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.484274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26a5ce39-d11d-43b9-86d4-74574360160c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.485121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:01.485407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:01.485674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:37:01.485956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:37:01.500563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:37:01.500863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a8b6e64-9718-464c-ac02-463d909ba1d0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.503542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[224f254d-b1a7-4785-944d-267562a4e69c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.504637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:01.528957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:01.530059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:01.531100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:01.531993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:01.532478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:01.577884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ee023f7-830b-41cd-9c3a-7bc9f5ac3437]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.614630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[80da5fa6-ec89-4067-863a-3b6fca126f3a]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:37:01.614630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:37:01.614630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:37:01.614630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:37:01.614630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:37:01.614630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:37:01.614630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:37:01.614630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:37:01.614630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:37:01.614630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:37:01.614630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:37:01.614630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:37:01.614630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:01.615904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80da5fa6-ec89-4067-863a-3b6fca126f3a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:01.616654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:01.618977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:01.621765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a37b9a0a-7d2f-49a8-893b-b14066baf619]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.621765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:01.621765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:01.621765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:37:01.621765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:01.622776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3008a259-7977-43b3-ac03-2f2052a04705]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.623747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[818660cd-c571-4045-960b-10915717956d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.625957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6153fa32-88bb-4c8c-ba40-f905e7fd0959]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.644723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56bb5ca8-00d1-4267-9e0c-20202066bde6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 311766, 'reachable_time': 42400, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143975, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.650996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:37:01.651252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a741ffa8-7d1a-42e7-82dd-19f880a60923]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:01.652315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd0fbd76-f17e-48d7-a3c6-89b4557b3537", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.113", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "72e3e169-abbc-435b-a035-9cfe735aacbf", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.9", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2061, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:f0:18", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1912bdb1-a7cc-4930-a1f3-dbfcaf7fe0d7", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:42Z", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:42Z", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:01.656224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:00Z", Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id dd2204c3-586b-4fe2-83ed-a05211a04e6d. Wait time elapsed: 17.505 Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id dd2204c3-586b-4fe2-83ed-a05211a04e6d. Wait time elapsed: 17.505 Jul 01 09:37:01.670341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:37:02.412179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.758s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:37:02.412800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:37:02.414344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:37:02.414736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:02.415244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:02.416612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac14eabf-f552-44e0-adc2-ae8e319fb267]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.430777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[383c725e-268f-45f0-8ec9-9a4a66fa0889]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.455724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4a9a94a-7163-46e8-9750-2985cf58c79f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.485848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce283be1-a92b-45e0-856c-24139a2e133a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 143978, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.506649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6765b662-23aa-44a5-9d27-d33e34787f70]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.527857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9580a390-d7a9-4dd8-9e30-0fdda1793eeb]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.548516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3233e470-a1ce-4b14-aff5-848503420346]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.569336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a04556a7-1fcd-480c-b768-6cb612ea5617]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.569934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:02.570214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:02.570502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:37:02.570969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:37:02.587788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:37:02.588057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[210d9c44-6e7b-4dae-94b2-8f150deb2fe5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.591290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be77ae02-b072-4d9a-9883-a6be4bc97fe0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.592387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:02.621171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:02.621961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:02.622365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:02.622757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:02.623147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:02.666669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4081eb49-a73f-4619-8999-1930c148f7c6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.694990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f7503414-bbc7-4e54-be99-a48edba9ea07]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:37:02.694990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:37:02.694990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:37:02.694990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:37:02.694990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:37:02.694990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:37:02.694990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:37:02.694990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:37:02.694990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:37:02.694990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:37:02.694990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:37:02.694990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:37:02.694990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7503414-bbc7-4e54-be99-a48edba9ea07]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:02.696289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:02.698539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:02.701371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a47c4acd-9401-426c-a01b-5e42f80b15b7]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.701371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:02.701371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:02.701371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:37:02.701371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:02.703609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de24466e-e51f-4c0c-9c90-37d6f41d495c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.704545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5d71909-27a8-4a7f-aa64-991f3df82749]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.705419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23558a9f-7f38-4d52-a60a-9011a4cd8669]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.728515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c33c59fe-55d8-46c3-9a24-3d41255a2949]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 311874, 'reachable_time': 39236, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 143997, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a8cb0cc-4bbf-4427-81b8-803e3e929a9a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:02.732566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd0fbd76-f17e-48d7-a3c6-89b4557b3537", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.113", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "72e3e169-abbc-435b-a035-9cfe735aacbf", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.9", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2061, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:f0:18", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1912bdb1-a7cc-4930-a1f3-dbfcaf7fe0d7", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:42Z", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:42Z", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:02.736591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:00Z", Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id dd2204c3-586b-4fe2-83ed-a05211a04e6d. Wait time elapsed: 18.586 Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id dd2204c3-586b-4fe2-83ed-a05211a04e6d. Wait time elapsed: 18.586 Jul 01 09:37:02.750755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:37:06.509720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.774s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:37:06.510299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:37:06.511183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:37:06.511549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:06.512064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:06.513186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dff0aff3-c045-465c-95f2-bd7a8e2c2364]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.536224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[809ea604-c1d8-43f4-8d88-f48eead4b4f8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.568130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d504090-749e-42f1-928f-32bde1c25dfd]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.608244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e155bd77-edd9-44e9-a7a5-1ed29dc1396e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 144100, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.629067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c854bb8-480c-4b7f-b131-c56a8ad07c76]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.655938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca5e0a87-6f81-4ee6-ba7a-1c5fc7bc158c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.676937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[813ebed0-2e6a-4880-825a-9dc89126fff5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.697113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46976d2a-3fd3-4b80-a117-d7366e68db65]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.697976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:06.698285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:06.698550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:37:06.698857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:37:06.715046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:37:06.715460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f00bebe3-abf7-4832-ab25-a34eebd6fba6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.718846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99e83fd7-7fa5-49de-942e-058722a08ace]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.719839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:06.750498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:06.750849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:06.751235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:06.751658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:06.752166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:06.795345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[087e8243-72fc-46d4-9a8a-bc90fb3009fd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.834439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ce5249a9-b156-4845-b773-d264cd005125]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:37:06.834439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:37:06.834439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:37:06.834439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:37:06.834439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:37:06.834439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:37:06.834439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:37:06.834439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:37:06.834439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:37:06.834439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:37:06.834439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:37:06.834439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:37:06.834439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:06.835683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce5249a9-b156-4845-b773-d264cd005125]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:06.836277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:06.838137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:06.841145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b846c085-62cf-4c45-a8ba-5bc5c4e03acf]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.841145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:06.841145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:06.841145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:37:06.841145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:06.842299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7730597-875c-42a4-a04b-89a97e88a9d8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.843397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f36add56-7690-4052-ad7a-e4959eca6f53]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.844425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acb8bf06-fed8-4108-add4-9faa1fc0e3a4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.863636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a571ea37-ff9e-457e-9348-82ffb52cee23]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 312287, 'reachable_time': 31476, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 144119, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.869893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:37:06.870145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[436447c8-5186-4478-9e4c-8d67594840b3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:06.871224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd0fbd76-f17e-48d7-a3c6-89b4557b3537", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.113", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "72e3e169-abbc-435b-a035-9cfe735aacbf", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.9", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2061, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:f0:18", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1912bdb1-a7cc-4930-a1f3-dbfcaf7fe0d7", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:42Z", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:42Z", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:06.876511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:04Z", Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id dd2204c3-586b-4fe2-83ed-a05211a04e6d. Wait time elapsed: 22.724 Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id dd2204c3-586b-4fe2-83ed-a05211a04e6d. Wait time elapsed: 22.724 Jul 01 09:37:06.892271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:37:07.718360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.845s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:37:07.719397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:37:07.720360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:37:07.720850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:07.721139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:07.722522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65deaee1-baed-4b9b-b4a1-7fcf25d79b55]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:07.739600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ec41114-17c6-47c9-abc6-72db2273e557]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:07.761639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0631560-fd36-41fe-95ad-bb1981ec7a54]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:07.792905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10aaad74-a622-4ad2-9e0f-b65358fd6d55]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 144122, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:07.817140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efae0497-9016-4d1b-aac0-5c2a78f64a93]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:07.839553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17838639-d10f-414e-b2d2-58b37d3443e8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:07.860695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[991f48ed-b511-4682-8ed5-b46e033ad178]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:07.881092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[daa79d93-33ad-448b-8ccc-2d547f986180]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:07.881651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:07.882005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:07.882434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:37:07.882743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:37:07.897666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:37:07.898080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86e8d97c-9d31-4fc3-b290-19e2199c8f60]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:07.901111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be3033b7-8833-4b91-9dfd-5f612ec42113]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:07.902221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:07.946189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:07.946875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:07.947272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:07.947693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:07.948079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:07.988628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bacd4356-87b6-48ba-8d68-d2ed0b3f2c9f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:08.021322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d17d5b6f-3aa0-449f-97e8-9f820ca9d824]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:37:08.021322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:37:08.021322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:37:08.021322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:37:08.021322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:37:08.021322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:37:08.021322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:37:08.021322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:37:08.021322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:37:08.021322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:37:08.021322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:37:08.021322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:37:08.021322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d17d5b6f-3aa0-449f-97e8-9f820ca9d824]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:08.022598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:08.024878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:08.027644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5194bb76-f0d3-42fe-9367-1fb51bdef310]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:08.027644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:08.027644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:08.027644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:37:08.027644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:08.028598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[126bdccc-e19e-4aab-8117-307315b2f2cf]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:08.029622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[659867da-2473-455b-9a09-c74a7007c2cf]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:08.031833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc70741b-7ac5-41f2-87a6-7a13ad2c9938]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:08.050249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b06d1dcc-200a-46b3-bdaa-98fb00cbbca6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 312405, 'reachable_time': 41796, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 144141, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:08.055627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:37:08.055946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6122f2f0-7389-4981-b133-3c100c16b7a8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:37:08.057060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd0fbd76-f17e-48d7-a3c6-89b4557b3537", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.113", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "72e3e169-abbc-435b-a035-9cfe735aacbf", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.9", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2061, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:f0:18", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1912bdb1-a7cc-4930-a1f3-dbfcaf7fe0d7", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:42Z", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:42Z", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:08.060652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:04Z", Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:37:08.074580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3 Jul 01 09:37:15.082956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:15.083688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:15.083688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:15.084165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:15.710507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-afdddb35-9bed-4777-b9b1-91ae7e95b97b tempest-TestVolumeBootPattern-830779102 tempest-TestVolumeBootPattern-830779102-project] Got routers updated notification :['9e1a30da-e2f9-4cd7-a55b-cc3a6e174576'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:37:15.712442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 1, update_id a4a4b933-e47f-4b54-abcf-715becd23cec. Wait time elapsed: 0.001 Jul 01 09:37:15.713023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 1, update_id a4a4b933-e47f-4b54-abcf-715becd23cec. Wait time elapsed: 0.002 Jul 01 09:37:15.713431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:37:18.731194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.018s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:37:18.731904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:37:18.733909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:37:18.734311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:18.734656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:18.736831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ef099a8-e4a0-43c0-a7bd-aaffefc146b5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:18.759526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42381000-466a-454a-be97-e6ce3ffbcc3d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:18.781875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dda3ce28-5475-4458-9a74-ad6481e7f663]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:18.810797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aad39a9b-11ef-4060-8770-52906d87d5a2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 144310, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:18.831827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbe7535a-bb3d-4881-9085-7b6648559793]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:18.852125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ee9d8f5-ae79-4f64-b514-686de23c16d7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:18.873921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfbfca6b-0ec0-4f9c-82ce-83f70af528d5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:18.893911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76d37537-06e3-41f8-ba8d-48f71ac1ae23]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:18.894504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:18.894780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:18.895052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:37:18.895328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:37:18.910244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:37:18.911001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27599dbc-ab74-4a79-aa92-efaee4e18827]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:18.913783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cbf096c-ea96-4c75-b660-11d1487f95c6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:18.914746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:18.947145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:18.947920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:18.948446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:18.948935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:18.949517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:18.988911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22fa4e3d-856d-4b60-b211-19a77e9ac3ef]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.020934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5a4bf16a-72d2-42b5-ab98-5b53b51b08f7]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:37:19.020934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:37:19.020934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:37:19.020934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:37:19.020934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:37:19.020934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:37:19.020934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:37:19.020934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:37:19.020934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:37:19.020934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:37:19.020934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:37:19.020934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:37:19.020934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:19.022191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a4bf16a-72d2-42b5-ab98-5b53b51b08f7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:19.022812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:19.025206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:19.028891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae7f8673-da5a-42b5-a978-85e756248d29]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.028891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:19.028891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:19.029258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:37:19.029529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:19.031151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ed5b4b3-3b61-435b-b080-0beade6ebf66]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.032233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[451990b2-18aa-4a1c-82de-2aa832fcb11d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.033165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3e0a1a2-7290-4c6f-b3ea-44f8e3cf8896]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.052787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5c3c607-43f8-452e-94e9-2aca19c8e0ed]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 313507, 'reachable_time': 42104, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 144328, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.057396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:37:19.057396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5576821-f2c5-4e47-8da0-8cf9b52368bc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:19.058534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "052eae5a-b4a9-4026-98c2-73610b3a823f", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.168", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "0d89369d-5dae-4985-82cd-24c6266f119e", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2080, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:8f:4c:e8", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9083073e-8161-47ca-bafd-5f6ef1a543db", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:37:14Z", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:14Z", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:19.062403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:59Z", Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 13, Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a4a4b933-e47f-4b54-abcf-715becd23cec. Wait time elapsed: 3.349 Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a4a4b933-e47f-4b54-abcf-715becd23cec. Wait time elapsed: 3.349 Jul 01 09:37:19.076262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:37:19.795135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.734s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:37:19.795940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:37:19.796657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:37:19.796995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:19.797409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:19.798803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a98dd0a5-af6f-47a9-ae9c-c30f29f4e0d6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.813509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bc470e9-a7b8-46fa-9b5a-dabd714b1d6e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.834191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18a202fa-8e05-433f-be20-348f175a77b7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.862109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22624405-98f8-4c48-8451-eb87b702de61]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 144331, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.882546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5c3dab6-b640-4608-926d-dd410d9f76b4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.903730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95edfb14-c099-495f-ab46-671ed45e005d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.924972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d10153d3-576b-4555-8bbd-e3e203472612]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.945178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[583806d1-af14-4de6-b6bf-48deb7db3141]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.945653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:19.945925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:19.946185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:37:19.946456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:37:19.961326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:37:19.961636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf604157-5e83-4ef7-b05e-756284f4486e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.964412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb49ffb8-6b2f-4e80-a1b6-85ae74b49a0f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:19.965364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:20.012776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:20.013440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:20.014469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:20.015086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:20.015546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:20.058366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9cb009e-d144-4c07-a3c8-0c6adcd5b8d1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:20.089837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[85e8d934-9a62-449e-b64e-66ed5afc338f]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:37:20.089837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:37:20.089837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:37:20.089837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:37:20.089837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:37:20.089837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:37:20.089837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:37:20.089837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:37:20.089837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:37:20.089837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:37:20.089837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:37:20.089837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:37:20.089837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:20.090947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85e8d934-9a62-449e-b64e-66ed5afc338f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:20.091043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:20.092985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:20.096539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[468cc379-f7d0-4e49-b7f8-d8f489bc4659]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:20.096539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:20.096539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:20.096539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:37:20.096539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:20.096539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a899aeb7-b714-4013-bca3-3a7506a6e361]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:20.097417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cce6e2e1-9d62-4810-a870-719ef5d7aa9d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:20.098458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d7639d5-3449-4a0f-b09d-df596eb5ffcb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:20.119247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35c5a803-5c53-459f-bd94-2ed8ad18f26d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 313612, 'reachable_time': 39132, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 144349, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:20.129550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:37:20.129841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1e4acc3-6092-4f2f-8d83-01bff328dfd5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:20.132196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "052eae5a-b4a9-4026-98c2-73610b3a823f", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.168", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "0d89369d-5dae-4985-82cd-24c6266f119e", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2080, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:8f:4c:e8", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9083073e-8161-47ca-bafd-5f6ef1a543db", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:37:14Z", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:14Z", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:20.135508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:59Z", Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 13, Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a4a4b933-e47f-4b54-abcf-715becd23cec. Wait time elapsed: 4.422 Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a4a4b933-e47f-4b54-abcf-715becd23cec. Wait time elapsed: 4.422 Jul 01 09:37:20.149811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:37:22.035057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.902s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:37:22.035697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:37:22.036433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:37:22.037314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:22.037683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:22.038949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d2b31ec-1c4e-446d-8505-b0521081fe41]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.056519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af5c03af-bc0a-4657-9b1f-2f23d7f7c310]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.080264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82380bea-526f-4196-9e05-83adaaf98e1d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.107117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88229dbf-cbe9-4646-82b8-a5a6589fc820]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 144369, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.127832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5333848e-45d1-4a17-857d-1e71d451293f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.147623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f1af5e7-d4c2-430f-82ea-ffea2b06f3e9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.162853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b606dc34-9173-4276-b889-9396a779d374]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.182638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3295553-ffc8-4c90-be6b-bd8e1a2e3cf5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.183291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:22.183880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:22.184235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:37:22.184579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:37:22.198984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:37:22.199390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d0bad1b-e354-45ce-b742-a5536b47175e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.203072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9247545-1681-473c-ae2c-55cd38cd3b3f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.204096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:22.233818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:22.234960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:22.235376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:22.235772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:22.236165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:22.270014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f58247f7-f026-466a-b5d5-b27a94dd006a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.312194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e035cb25-3ab9-497d-8481-06c73e8ad358]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:37:22.312194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:37:22.312194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:37:22.312194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:37:22.312194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:37:22.312194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:37:22.312194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:37:22.312194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:37:22.312194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:37:22.312194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:37:22.312194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:37:22.312194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:37:22.312194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e035cb25-3ab9-497d-8481-06c73e8ad358]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:22.314264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:22.318625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:22.324288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da1f81aa-ee50-4600-a7d1-8022af5d9590]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.324288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:22.324288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:22.324288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:37:22.324288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:22.324288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2dfbec70-1eae-45b3-8d01-18c72de48a0b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.324288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2640a88b-055d-4b71-9201-9a51e76f9939]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.325588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10dbabdb-b16c-48f2-ac57-7a8b025099c6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.343780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09ade909-5131-4a08-b93a-a3d8116ff3bf]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 313837, 'reachable_time': 37428, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 144387, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.349885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:37:22.350144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e26732f-f973-4bc3-8343-c9fee9f761c9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:22.351695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "052eae5a-b4a9-4026-98c2-73610b3a823f", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.168", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "0d89369d-5dae-4985-82cd-24c6266f119e", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2080, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:8f:4c:e8", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9083073e-8161-47ca-bafd-5f6ef1a543db", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:37:14Z", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:14Z", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:22.355179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:20Z", Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a4a4b933-e47f-4b54-abcf-715becd23cec. Wait time elapsed: 6.642 Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a4a4b933-e47f-4b54-abcf-715becd23cec. Wait time elapsed: 6.642 Jul 01 09:37:22.371797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:37:23.198504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.845s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:37:23.199134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:37:23.199844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:37:23.200241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:23.200595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:23.202344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40ec1adc-b4ef-4f7e-afd8-640059fcbf56]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.218028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90361403-9ec2-4c4a-a374-9324acd50136]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.236520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d840ee06-f9c8-4bb4-923a-385b5d0e3b17]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.268917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6947506-767e-41b8-ab3a-7e7c9e2bab1e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 144390, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.291998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[389dd2b1-ef56-4a0e-9e86-d76e5cfddfc2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.313408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fbb4f2b-d0d8-4281-b10d-44e437156675]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.335258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6f77617-318b-46d6-a061-085c62054d6e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.350060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb7f83df-e515-42cb-8b78-4cffafd33807]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.351158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:23.352155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:23.352530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:37:23.353651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:37:23.371910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:37:23.372674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3afe045-b9e4-44b3-ae18-9f6192b68bad]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.376707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa2f5c25-7edc-4f28-bcbf-9644f62ae788]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.377796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:23.412339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:23.413279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:23.422618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:23.423151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:23.423587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:23.551512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8fbe4d5-6c2d-44a6-8ffb-b805fd2e9e90]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.584107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c17e0835-bf77-4cae-9556-7b4a220da0dd]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:37:23.584107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:37:23.584107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:37:23.584107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:37:23.584107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:37:23.584107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:37:23.584107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:37:23.584107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:37:23.584107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:37:23.584107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:37:23.584107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:37:23.584107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:37:23.584107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c17e0835-bf77-4cae-9556-7b4a220da0dd]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:23.585343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:23.587694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:23.591342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[754aa538-8c34-4f69-b46a-6c88bca5f885]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.591342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:23.591342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:23.591342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:37:23.591342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:23.591342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78bad3e2-f455-4038-a85f-132d8a36c785]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.592544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[759b26b7-33b0-4322-b191-8db96507f84f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.594179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b5e61e4-c8ba-429c-8345-b8d300b90e55]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.614999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40512dde-0712-4a26-8c9e-48c2cf6312f3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 313953, 'reachable_time': 19280, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 144409, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.620205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:37:23.620205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13b4b38d-2ccf-4631-a57a-2d73524e79fa]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:23.621825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "052eae5a-b4a9-4026-98c2-73610b3a823f", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.168", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "0d89369d-5dae-4985-82cd-24c6266f119e", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2080, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:8f:4c:e8", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9083073e-8161-47ca-bafd-5f6ef1a543db", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:37:14Z", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:14Z", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:23.626062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:22Z", Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a4a4b933-e47f-4b54-abcf-715becd23cec. Wait time elapsed: 7.912 Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a4a4b933-e47f-4b54-abcf-715becd23cec. Wait time elapsed: 7.912 Jul 01 09:37:23.641182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:37:27.296154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.672s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:37:27.296747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:37:27.297820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:37:27.298178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:27.298465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:27.299668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af08a412-5757-4565-b38c-856df2997620]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.315427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9631f2d6-b7c3-4e1a-9c92-0261cc360a08]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.335491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a90dc00f-448c-42e6-af22-07fd41aea78a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.363899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b02c7fd0-c631-4a4f-8787-9f7ebf1cd24e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 144477, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.384730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05aca0b4-ed33-4edb-bf3e-b5683c5ebe4a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.405148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f508a2f-8b14-4abd-9022-8a54247e55e1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.425547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c4a298e-e1ca-48aa-901a-529dd7347b4c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.450644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cfccaa1-c1f6-4646-b58f-a9cf26d23902]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.451280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:27.451721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:27.452162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:37:27.452493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:37:27.469167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:37:27.469466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86584d45-10fd-4059-b0c0-95651f237f05]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.472216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f632135c-9939-4f6d-a816-d818cbc64379]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.473189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:27.501530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:27.502769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:27.503482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:27.503880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:27.504548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:27.549660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76cefbe1-228b-480c-81c0-57b093aafe31]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.580563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6f63b8d7-3a23-44af-b826-f459b9aa3f83]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:37:27.580563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:37:27.580563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:37:27.580563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:37:27.580563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:37:27.580563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:37:27.580563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:37:27.580563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:37:27.580563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:37:27.580563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:37:27.580563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:37:27.580563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:37:27.580563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f63b8d7-3a23-44af-b826-f459b9aa3f83]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:27.581918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:27.584174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:27.587346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5949ca61-99cc-4b3f-9477-0bd7441d2052]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.587346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:27.587346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:27.587783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:37:27.588181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:27.589268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0de60491-cc92-44b7-b64a-0ec2d252fa56]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.590211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd956725-40c4-44f6-92c0-715ff2e8de68]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.591466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bfb4b31-5305-4046-9bd6-ecd577f92d42]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.612772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a6beaf9-9824-45a6-a97a-6ce078f5a20a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 314362, 'reachable_time': 20132, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 144495, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.614920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:37:27.615346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9b4e153-1bb4-4a4a-8d21-eff634cacd59]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:27.616581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "052eae5a-b4a9-4026-98c2-73610b3a823f", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.168", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "0d89369d-5dae-4985-82cd-24c6266f119e", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2080, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:8f:4c:e8", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9083073e-8161-47ca-bafd-5f6ef1a543db", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:37:14Z", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:14Z", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:27.620854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:24Z", Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a4a4b933-e47f-4b54-abcf-715becd23cec. Wait time elapsed: 11.910 Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a4a4b933-e47f-4b54-abcf-715becd23cec. Wait time elapsed: 11.912 Jul 01 09:37:27.641073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:37:28.380442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.756s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:37:28.381038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:37:28.381765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:37:28.382100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:28.382523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:28.383725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ba10912-0ecb-4880-8e82-1028703ddece]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.398085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb53094a-b1ee-464a-a78c-c38162b3f8ec]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.424065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8fba96df-70dc-4f17-b9d8-87f87e6da118]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.470235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2facf57a-ef2d-471f-b069-e6de2e568b57]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 144498, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.491764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c5a65f5-d692-4e64-ae74-a3e9fddf70c0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.514302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4e920be-b047-4a94-a1d8-0b4085d89f27]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.536400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88dcbac6-b8d6-44f0-b498-b4855a1b36d1]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.558914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae81097f-4d01-49e4-969c-00c1faa972d1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.560433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:37:28.561856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:37:28.562574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:37:28.563079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:37:28.593485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:37:28.593973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c151c657-b617-4f8b-a1a3-b74b50441a99]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.598918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04c97577-165a-42f6-a231-15bc083892a5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.599998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:28.638814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:28.638814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:28.638814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:28.639178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:28.639764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:37:28.691870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f1969dd-b663-453e-9da3-f44c1c934890]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.737018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[260a3023-f081-4b1a-8e18-88b21c8bb671]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:37:28.737018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:37:28.737018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:37:28.737018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:37:28.737018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:37:28.737018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:37:28.737018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:37:28.737018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:37:28.737018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:37:28.737018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:37:28.737018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:37:28.737018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:37:28.737018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:28.738248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[260a3023-f081-4b1a-8e18-88b21c8bb671]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:28.738356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:28.740750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:28.743496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60163df5-5b8c-41f8-9d17-f746920c593d]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.743496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:28.743496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:28.743496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:37:28.743496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:37:28.743982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73469044-ca03-4514-bdf9-454613bdb4b1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.744582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9acd94b6-3f9b-48e0-8ba2-17a66cea3fbc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.745660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc65883c-be9f-489f-a547-3edf1afc13c5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.767188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c977f036-144b-4f5d-878a-c1f2260f26f7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 314473, 'reachable_time': 19104, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 144532, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.769163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:37:28.769536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78e2087c-bcb3-4fe4-8c5b-2340b8be67e0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:37:28.770840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "052eae5a-b4a9-4026-98c2-73610b3a823f", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.168", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "0d89369d-5dae-4985-82cd-24c6266f119e", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2080, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:8f:4c:e8", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9083073e-8161-47ca-bafd-5f6ef1a543db", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:37:14Z", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:14Z", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:37:28.774545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:26Z", Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:37:28.789196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3 Jul 01 09:37:35.123949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:38:15.083273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:38:15.083912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:38:15.083912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:38:15.084198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:38:15.125392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:38:55.127053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:39:15.084031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:39:15.084732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:39:15.084732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:39:15.085041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:39:35.128568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:40:15.084922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:15.085550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:15.085550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:15.085809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:15.130448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:40:47.467967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-827f75ae-3cb3-4ce2-9cac-ecf357950f50 tempest-TestVolumeBootPattern-830779102 tempest-TestVolumeBootPattern-830779102-project] Got routers updated notification :['9e1a30da-e2f9-4cd7-a55b-cc3a6e174576'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:40:47.468863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 1, update_id a0b0dd54-85e6-4d92-8482-366dafd2b78c. Wait time elapsed: 0.001 Jul 01 09:40:47.469348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 1, update_id a0b0dd54-85e6-4d92-8482-366dafd2b78c. Wait time elapsed: 0.001 Jul 01 09:40:47.469734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:40:48.842117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.372s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:40:48.843600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:40:48.845703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:40:48.845840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:48.846293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:48.853960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8021baa-3a5a-4e44-a765-0eb11c313926]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:48.888688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4c6c58f-d762-4999-926d-49b4f06bf9bf]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:48.931391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2f0b640-2e98-4255-b015-e72b50908ab6]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:48.961416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3250fabc-a3dc-49da-a4d5-729134526a34]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 147714, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:48.988892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31676400-7ed9-4aab-90ce-3dd19338029b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.013873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4e5bf9e-10cb-437d-9314-fb6d71294901]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.036624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26ac0c89-9ef9-4b55-b8e6-81af0f1eb903]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.061043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9901c3f9-0079-41e4-96fa-0f5bb1133a53]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.061633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:49.062164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:49.062442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:40:49.062744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:40:49.080326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:40:49.080624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c7149a3-0373-42fc-8cf0-fa2767abbdba]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.083314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[916bbbf8-f137-4c48-a812-e6a7141a45e2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.084337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:49.112980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:49.114159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:49.114621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:49.115013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:49.115421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:49.147434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b6dcf4f-8703-4509-9a7c-b398d6374b55]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.178058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7efbe0b4-3a32-4bae-8ab3-d3d37fc90490]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:40:49.178058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:40:49.178058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:40:49.178058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:40:49.178058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:40:49.178058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:40:49.178058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:40:49.178058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:40:49.178058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:40:49.178058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:40:49.178058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:40:49.178058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:40:49.178058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7efbe0b4-3a32-4bae-8ab3-d3d37fc90490]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:49.179375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:49.181694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:40:49.185200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2136f042-3c53-4d0e-91b1-7724712a452e]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.185200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:49.185200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:49.185200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:40:49.185629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:49.186454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[435b141a-402c-4530-b5a0-d43f6ad86c69]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.187547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12f5c359-ed65-4161-83bc-a6e3d4d1af53]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.188574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c500bc73-4884-402b-8ba1-521e59a7ce90]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.207015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[951c15fc-c40b-40b0-b83c-523172b897a6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 334522, 'reachable_time': 37612, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 147732, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.208911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:40:49.209216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9fe02a91-a4f0-4e0f-9040-861897456d27]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:49.210226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:32Z", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:40:49.214693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a0b0dd54-85e6-4d92-8482-366dafd2b78c. Wait time elapsed: 1.743 Jul 01 09:40:49.234493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a0b0dd54-85e6-4d92-8482-366dafd2b78c. Wait time elapsed: 1.744 Jul 01 09:40:49.234493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:40:50.024800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.813s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:40:50.025364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:40:50.026249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:40:50.026745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:50.027091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:50.028519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6beed19f-e79d-4e2a-9819-2567c8f907d4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.048006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9962c464-8c7a-4c75-9fd2-acd1f50a4724]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.104467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bbf57a4-9eaa-47a5-8a53-23d976289502]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.158035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0fc881b-03cd-4a24-80fe-0f3a0f5cf508]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 147737, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.190127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d16741c6-d9ad-427d-944c-0d5ea512d284]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.214927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2313fb6-03f5-4c6a-9bd8-8bae3abc7371]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.244561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6bd3ac7-a842-450d-8c71-669586b8bbd5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.270673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fcf44e3-dc81-41f7-8190-53d735eed2cb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.271253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:50.271857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:50.272129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:40:50.272411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:40:50.288546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:40:50.289196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f7508f6-8a50-4eb2-8919-98c8d978bf91]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.292568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1e3c0f5-1fe6-4c37-8e06-c579c1cd610f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.294893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:50.334612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:50.335645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:50.336175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:50.336688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:50.337209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:50.391719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50fc135b-6e5a-452f-bd92-219f48ff6b7b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.426044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[78162da6-f90f-4847-a2a6-e57077dbec0d]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:40:50.426044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:40:50.426044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:40:50.426044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:40:50.426044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:40:50.426044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:40:50.426044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:40:50.426044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:40:50.426044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:40:50.426044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:40:50.426044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:40:50.426044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:40:50.426044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:50.426795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78162da6-f90f-4847-a2a6-e57077dbec0d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:50.427479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:50.432328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:40:50.436507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[120d1075-f953-4a68-8261-acc37268376e]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.438370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:50.438909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:50.439346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:40:50.439998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:50.441310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94f4707c-2d70-4fb1-84c1-113dc6711996]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.442404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c5c1361-e517-4e49-b03e-e75588a6396b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.443620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e27a84e-148c-421b-be80-3e733b74ddbb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.469230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e4bd996-ad3d-4a94-9849-aeaa082279e1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 334642, 'reachable_time': 33236, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 147758, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.471484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:40:50.471743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44e1e832-64bd-4f00-aa9a-1e754df5c877]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:50.472754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:32Z", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:40:50.474789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a0b0dd54-85e6-4d92-8482-366dafd2b78c. Wait time elapsed: 3.006 Jul 01 09:40:50.489463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a0b0dd54-85e6-4d92-8482-366dafd2b78c. Wait time elapsed: 3.006 Jul 01 09:40:50.489463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:40:53.666678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.192s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:40:53.667273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:40:53.668079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:40:53.668549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:53.668989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:53.671254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a595b1ab-544a-4bf1-b231-d76f6ba3802d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:53.688133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eaad7b74-9e8b-43ee-ab19-ece4d5872882]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:53.711872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d55a8a07-e3df-4294-988a-e80d5b275d0d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:53.746489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4b663bc-9a78-47c9-963d-49191a1115a6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 147843, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:53.768344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8068bcc4-3366-4fc3-beb9-3973f499dc23]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:53.790132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44f46edc-ae3c-4198-a2f9-b1dc3742c943]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:53.811110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbe40676-9bdf-4327-b20e-456358f7d998]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:53.832843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44c921f2-c698-457b-a5d9-2bfce8e903fd]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:53.833487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:53.833823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:53.834150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:40:53.834523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:40:53.849191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:40:53.850228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80e4e72b-4c5f-42e6-baaa-532edc35de44]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:53.852979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e07b6a67-a2f7-44b6-979f-51d15d5f5796]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:53.854032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:53.888976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:53.890092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:53.891350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:53.891980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:53.892549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:53.971581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[086772f9-22d4-4d01-a98e-2e1a8c90fc14]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:54.004528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[79c10b8a-c248-4fb9-bd28-6b29abcd518c]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:40:54.004528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:40:54.004528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:40:54.004528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:40:54.004528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:40:54.004528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:40:54.004528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:40:54.004528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:40:54.004528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:40:54.004528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:40:54.004528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:40:54.004528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:40:54.004528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:54.005319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79c10b8a-c248-4fb9-bd28-6b29abcd518c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:54.006758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:54.008080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:40:54.009631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a3ecba7-1011-4766-bafa-7ca5100a3258]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:54.009780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:54.010244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:54.010688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:40:54.011166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:54.012937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06484bb4-b8ee-4292-884d-c749a565c7d9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:54.014353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d1525a1-e1b4-4096-9dcc-abcfb1f9649b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:54.015483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cea32cdd-7a96-425b-bfde-f00f5cb0613d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:54.036802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e6727b7-4d57-4869-b9cb-ca90e7ed25e0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 335000, 'reachable_time': 41784, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 147863, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:54.042715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:40:54.042715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2390ffad-8122-42f1-9ced-66773e33e45a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:54.044097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:40:51Z", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 24, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:40:54.047160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:40:54.059990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a0b0dd54-85e6-4d92-8482-366dafd2b78c. Wait time elapsed: 6.590 Jul 01 09:40:54.059990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a0b0dd54-85e6-4d92-8482-366dafd2b78c. Wait time elapsed: 6.590 Jul 01 09:40:54.059990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:40:54.945316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.886s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:40:54.946149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:40:54.947346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:40:54.948156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:54.948450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:54.949609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf0596c3-6e8f-4490-a725-2bf673c0aa86]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:54.971695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa76f26a-a2b0-43f9-bf65-65804bfac036]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.005276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[389d149a-9001-41a7-81c5-36ed02ab64ce]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.036174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97363f09-a3b1-4f5d-b826-dd1084470589]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 147869, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.062200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28017039-9bd6-46a9-953e-d8736f684afc]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.090298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb7620af-6184-4301-93f7-1bf4123a7d7a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.115746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71efcf0b-76cf-4357-9eb1-0bcfa964a9c9]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.131687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:40:55.139001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b81451ef-1b97-432c-8596-52beded14a3d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.139646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:55.139924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:55.140189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:40:55.140468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:40:55.162161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:40:55.162886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83106f54-24f6-49ae-8371-d3f312e9c44c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.166606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df6beb38-b970-416b-90a9-da0e7167a044]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.167681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:55.207065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:55.209882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:55.210846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:55.211238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:55.211655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:55.248214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c0f5bb4-2349-45cb-a516-7b3d673cd015]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.289974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[900effc2-a4e4-48c6-b795-2d25f03ad94c]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:40:55.289974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:40:55.289974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:40:55.289974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:40:55.289974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:40:55.289974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:40:55.289974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:40:55.289974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:40:55.289974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:40:55.289974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:40:55.289974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:40:55.289974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:40:55.289974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:55.290821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[900effc2-a4e4-48c6-b795-2d25f03ad94c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:55.291147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:55.292621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:40:55.295392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[844de12b-0363-4da3-9b0a-9d3c8276eae9]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.296445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:55.296937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:55.297348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:40:55.298037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:55.299324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2929d699-09fc-494d-a904-f62b45c2860b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.300381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f283c94-8aac-4792-b4bf-d46f258a0772]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.301382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1aca493-e7b2-423d-8d3f-d9ae41ab0a40]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.326656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32021ace-8adf-4b60-9ee8-913c8a370329]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 335130, 'reachable_time': 16868, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 147890, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.332468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:40:55.332770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cfe91150-ea07-475b-890a-cbb12e2c079d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:55.333793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:40:53Z", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 25, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:40:55.336081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a0b0dd54-85e6-4d92-8482-366dafd2b78c. Wait time elapsed: 7.867 Jul 01 09:40:55.344855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a0b0dd54-85e6-4d92-8482-366dafd2b78c. Wait time elapsed: 7.867 Jul 01 09:40:55.344855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:40:56.810729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.475s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:40:56.811284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:40:56.812051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:40:56.812434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:56.812909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:56.814608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f5f4693-4886-4103-9baf-74f984dddf4f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:56.831534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2040373-102f-4e68-9c02-b1d81e8f9c54]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:56.851960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15c7bf6d-f02d-4a3b-b24c-66285d51150c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:56.885514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6447a007-7146-405f-b7e1-1ee743c4078f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 147918, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:56.908661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e418497e-89f7-4815-a866-e6772a8aa599]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:56.930327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f893478-b4d6-4258-8125-ac56670d1a90]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:56.968995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e7e7cee-132a-41ca-8824-437e653550d6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:56.985008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5f3a880-bbe0-44a2-89c1-1a6983bc6954]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:56.985995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:56.986277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:56.986550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:40:56.986824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:40:57.002316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:40:57.002603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc16f3ad-165c-4874-bd9c-4b32ed8c7f8a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.005616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c48f87b-e814-486f-9a27-b8c43054044b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.006863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:57.038536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:57.039198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:57.039614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:57.039999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:57.040381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:57.082202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[601cf757-727a-4ae3-99d1-a1cca257e0fe]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.112310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d88824c2-3c15-4ebc-98b1-bb99c1cf9fb4]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:40:57.112310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:40:57.112310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:40:57.112310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:40:57.112310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:40:57.112310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:40:57.112310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:40:57.112310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:40:57.112310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:40:57.112310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:40:57.112310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:40:57.112310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:40:57.112310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:57.113099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d88824c2-3c15-4ebc-98b1-bb99c1cf9fb4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:57.113541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:57.114834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:40:57.116457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a61b06fa-e11f-4574-a6b6-d78c26c010b2]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.116848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:57.117390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:57.117794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:40:57.118137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:57.119222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba2c4106-8e50-4ec5-880c-152db9e22d32]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.120306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[914e30ca-68bf-4c7d-9ac2-b4b401a278a8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.121215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21215c02-7145-478b-84c3-2359113b992d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.143108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55b41a7c-f51a-4c56-855c-b4ced173ab0f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 335314, 'reachable_time': 28700, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 147936, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.150085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:40:57.150402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d74c85f-e931-44c0-954d-7ba4155ee613]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:57.151509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:40:54Z", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 26, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:40:57.154262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:40:57.163701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a0b0dd54-85e6-4d92-8482-366dafd2b78c. Wait time elapsed: 9.695 Jul 01 09:40:57.163701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 2, update_id a0b0dd54-85e6-4d92-8482-366dafd2b78c. Wait time elapsed: 9.696 Jul 01 09:40:57.165090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:40:57.865002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.701s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:40:57.865872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:40:57.866926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9e1a30da-e2f9-4cd7-a55b-cc3a6e174576), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:40:57.867399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:57.867939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:57.870743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e6425c4-a84e-41a4-95d6-a1a2cec8b56b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.896114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49f60a3b-d531-4380-bd17-b5e66a64d779]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.920971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c7618ac-45e2-4a39-adf8-439c8c272d10]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.968289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[980633d6-719f-4d6b-9eb4-041e064a22ab]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 147945, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:57.989774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[423ae3aa-03dd-4f23-8697-e4ffd55727a0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.032216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f6b9c23-09fc-493f-b075-02cd8cf4c7bd]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.071386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a9bc67c-6f5f-437c-a539-aa2a9d2cfbf1]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.113117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fbe0055-2dbc-476b-8656-5318b46dace2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.114163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:40:58.114727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:40:58.115413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:40:58.115972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:40:58.137781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:40:58.138708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93f2408b-f999-4944-8697-a41f01c1fa5e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.141855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf3d2e9a-b145-4f2e-9d0f-3c96c3fb0037]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.143115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-bff0c8e3-2b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:58.180576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-bff0c8e3-2b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:58.181243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:58.181672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:58.182034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-bff0c8e3-2b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:58.182538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-bff0c8e3-2b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c8:68:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:40:58.227124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f52c426-02d3-4af1-b161-b1ef46aaae64]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.287981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1e7eae49-f6c4-46b2-a0cf-841b06fbc950]: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:40:58.287981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:40:58.287981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:40:58.287981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:40:58.287981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:40:58.287981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:40:58.287981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:40:58.287981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:40:58.287981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:40:58.287981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:40:58.287981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:40:58.287981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:40:58.287981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:58.290287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e7eae49-f6c4-46b2-a0cf-841b06fbc950]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:58.291671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:58.296855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:40:58.300439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2cb5019c-c771-42fd-b6b6-5e06aa546bb1]: (4, ['qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.301474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:58.301999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:58.302412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:40:58.302957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9e1a30da-e2f9-4cd7-a55b-cc3a6e174576.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:40:58.304312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c89845cb-9450-45e7-afe6-2078ac276172]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.305465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e46f12d-588d-4c3c-a97d-f4e1eb2c3d25]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.307216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd591527-5006-4a6b-916b-046c07633e45]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.337291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee630831-17aa-4121-97de-b95f3253dcad]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 335423, 'reachable_time': 31184, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 147966, 'error': None, 'target': 'qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.351509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:40:58.352009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92045a18-5d81-48fb-9948-0f1650e77aff]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-bff0c8e3-2b not found in namespace qrouter-9e1a30da-e2f9-4cd7-a55b-cc3a6e174576. Jul 01 09:40:58.353239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestVolumeBootPattern-715130324-router", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "782e9fbf-3821-41cc-853b-e44a675ba6ac", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:28:e1", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2015, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:13Z", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:19Z", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "bff0c8e3-2bfc-42b3-883d-6ecd0d9f5f72", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "434c7193-1c6c-4a84-b87a-48cc78ea2dd5", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c8:68:e8", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9e1a30da-e2f9-4cd7-a55b-cc3a6e174576", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2018, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:15Z", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:40:56Z", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 27, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7ac1bd5ac6534a26a28a1438a0863819", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dddd6da2-ebd1-4f15-984f-698879c9be86", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:40:58.360910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:40:58.374724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3 Jul 01 09:41:01.244600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-340bbbd6-0414-471e-bca5-2343b0669f27 admin admin] Got routers updated notification :['9e1a30da-e2f9-4cd7-a55b-cc3a6e174576'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:41:01.245190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 1, update_id c0277562-3a8c-4efa-93c2-474ed231a8a8. Wait time elapsed: 0.000 Jul 01 09:41:01.245640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 3, priority 1, update_id c0277562-3a8c-4efa-93c2-474ed231a8a8. Wait time elapsed: 0.000 Jul 01 09:41:01.245996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:41:02.422891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-7a1ab410-1bcc-47db-84c1-6fe38f165b64 admin admin] Got router deleted notification for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:41:02.775539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.529s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:41:02.775972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 was not found. Performing router cleanup Jul 01 09:41:02.777565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[273315c4-c165-4a97-874f-d7f78cc0581c]: (4, ['qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:02.778336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:02.779256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:02.779671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, update_id c0277562-3a8c-4efa-93c2-474ed231a8a8. Time elapsed: 1.535 Jul 01 09:41:02.780091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 1, priority 1, update_id 8f2795ed-f30f-4388-a2f4-866ad3f50b15. Wait time elapsed: 0.357 Jul 01 09:41:02.780494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, action 1, priority 1, update_id 8f2795ed-f30f-4388-a2f4-866ad3f50b15. Wait time elapsed: 0.357 Jul 01 09:41:02.781424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576 was not found. Performing router cleanup Jul 01 09:41:02.783517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7befaa3-634b-4c72-bfc1-1044b90c2350]: (4, ['qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:02.784364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:02.784805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:02.785225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9e1a30da-e2f9-4cd7-a55b-cc3a6e174576, update_id 8f2795ed-f30f-4388-a2f4-866ad3f50b15. Time elapsed: 0.005 Jul 01 09:41:04.396983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-f3bc2dbd-780b-44b5-aa62-4e35b3642e46 admin admin] Got routers updated notification :['b361ccc3-7577-4438-b123-f16e00901596'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:41:04.401185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 1, update_id 43dff3b8-6cc5-4ea6-a9e9-efa7f496c0ba. Wait time elapsed: 0.000 Jul 01 09:41:04.401414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 3, priority 1, update_id 43dff3b8-6cc5-4ea6-a9e9-efa7f496c0ba. Wait time elapsed: 0.004 Jul 01 09:41:04.401838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:41:05.036287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.634s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:41:05.036748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router b361ccc3-7577-4438-b123-f16e00901596 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:41:05.038382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:41:05.039328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:05.039640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:05.040977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[218ebe6e-dfff-4af3-ae45-837f8f089621]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.055853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f8b1c96-c957-40e4-99af-abfa1e38021b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.080396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b0d65ed-a923-4578-8b7e-a85966489428]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.107454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce6c135a-41aa-4562-be2e-76c61547c42a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 148063, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.128560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49fbb2c6-b96f-41e2-9e7b-03746eb2d7ef]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.149912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce762d8a-ea4a-4dca-943c-824122756700]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.172016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2d2fdc5-91b1-4e42-bd26-1b85f74f2f2c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.194546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[975e30af-864a-4485-9d71-d2e605e62f56]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.195109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:05.195404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:05.195683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:41:05.217142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c46d61f4-59b4-4f26-882a-c37bb70ff479]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 336136, 'reachable_time': 36900, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 148069, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.217847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:05.218135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:05.218580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:41:05.218812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:41:05.233202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-0e41a9a8-57e5-4e54-80cc-88413e22afcc admin admin] Got router deleted notification for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:41:05.242403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd50b85a-a81f-4911-ac17-6d2cfbe68427]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 336136, 'reachable_time': 36900, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 148070, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.242403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:41:05.242637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:41:05.283738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e53ed5a5-33b8-48b4-90bf-c34f497b9999]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.450987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a502dfc-025e-4bb0-b8bc-78e5e753c21b]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.473603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d7c6dbc-74ee-4312-90dc-0742cff9806b]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.671339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[635d5e6e-036b-4cdb-9bfb-fc6bfca9bd21]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.672256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:41:05.672696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:41:05.673091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:41:05.673636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" released by "process_external" :: held 0.455s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:41:05.674276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:41:05.674610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:41:05.675114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:41:05.675679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:41:05.697807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05f0ab31-b36d-46e6-b252-5cf95065fe8e]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.730392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[766bee14-53e4-4f5b-b2b6-5490b84098d7]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.744102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[676f913e-2fbf-40c5-96f3-2b66800fabac]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.782210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b190d2b-5285-4abe-8c74-5e438d8b0537]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.782858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:41:05.783256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:41:05.783608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:41:05.784139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" released by "process_address_scope" :: held 0.109s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:41:05.784606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (b361ccc3-7577-4438-b123-f16e00901596), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:41:05.785141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:41:05.786082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:41:05.786082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:41:05.786999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:41:05.805426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65fee9e4-adce-4430-a981-7dc38049bc7c]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.875601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe4a5042-2137-4aa2-a648-4bb005e025e7]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.889380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[487af357-9526-452f-aadc-0ec3cd0d1f4b]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:05 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:41:05 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.942714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6adb97dc-fac8-4463-8325-d0c2c31c1737]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:05.943345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:41:05.944050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:41:05.944506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:41:05.945007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-b361ccc3-7577-4438-b123-f16e00901596" released by "apply_metadata_nat_rules" :: held 0.160s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:41:05.945540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-b361ccc3-7577-4438-b123-f16e00901596 Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.haproxy Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID b361ccc3-7577-4438-b123-f16e00901596 Jul 01 09:41:05.945973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:41:05.948877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/b361ccc3-7577-4438-b123-f16e00901596.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:41:05.970561 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-b361ccc3-7577-4438-b123-f16e00901596[148086]: Proxy listener started. Jul 01 09:41:05.994881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router b361ccc3-7577-4438-b123-f16e00901596 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:41:05.995222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for b361ccc3-7577-4438-b123-f16e00901596, update_id 43dff3b8-6cc5-4ea6-a9e9-efa7f496c0ba. Time elapsed: 1.598 Jul 01 09:41:05.995581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update b361ccc3-7577-4438-b123-f16e00901596, action 1, priority 1, update_id 116ca416-db5e-4334-a08f-874653cc6e18. Wait time elapsed: 0.762 Jul 01 09:41:05.995827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for b361ccc3-7577-4438-b123-f16e00901596, action 1, priority 1, update_id 116ca416-db5e-4334-a08f-874653cc6e18. Wait time elapsed: 0.762 Jul 01 09:41:05.996175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router b361ccc3-7577-4438-b123-f16e00901596 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:41:05.997072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (b361ccc3-7577-4438-b123-f16e00901596), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:41:06.010747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11690be2-8ac6-4cb0-bc12-0ead66ef8ceb]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:06.013813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57fb3fee-7c5e-4410-b5b7-3228d28fa10c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:06.015313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a71260bb-fedf-4b02-b4f1-2ecce9b36137]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:06.015313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:41:06.016325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e98a2c4-1696-47c0-afde-4e10cd3db8ed]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:06.038059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f50c3f9-b01a-4723-ae76-83f45d8b92fc]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 336136, 'reachable_time': 36900, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 148092, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:06.039326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:06.039650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:06.059960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70bb09da-4282-4cf2-938b-012fe72f76cc]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 336136, 'reachable_time': 36900, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 148093, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:06.061416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:41:06.064005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:06.064005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/b361ccc3-7577-4438-b123-f16e00901596.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:06.064144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:41:06.064897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router b361ccc3-7577-4438-b123-f16e00901596 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:41:06.066376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28812ae7-6f96-4f5d-8c58-df85020962c6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:06.088571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6c282e2-a733-42e7-add4-515336143426]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 336136, 'reachable_time': 36900, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 148094, 'error': None, 'target': 'qrouter-b361ccc3-7577-4438-b123-f16e00901596', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:06.099342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-b361ccc3-7577-4438-b123-f16e00901596 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:41:06.099632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b10ff97-b698-4e52-bc19-b7205d30029a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:06.100560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info b361ccc3-7577-4438-b123-f16e00901596 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:41:06.101257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (b361ccc3-7577-4438-b123-f16e00901596), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:41:06.101845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:06.102125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:06.102819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:06.103280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:06.103710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for b361ccc3-7577-4438-b123-f16e00901596, update_id 116ca416-db5e-4334-a08f-874653cc6e18. Time elapsed: 0.108 Jul 01 09:41:15.085998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:15.086642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:15.086642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:15.086816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:35.133724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:41:46.128416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-2d7f278c-232b-4bf9-b0fc-8da1f16fc81d tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['203e4741-69b9-468f-97b4-76a7d63b11b8'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:41:46.130078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 1, update_id 604e940a-6986-426c-8b0d-3019a27f37be. Wait time elapsed: 0.001 Jul 01 09:41:46.130668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 1, update_id 604e940a-6986-426c-8b0d-3019a27f37be. Wait time elapsed: 0.001 Jul 01 09:41:46.131172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:41:48.297968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.165s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:41:48.298525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:41:48.300322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:41:48.300834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:48.301559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:48.305278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ca88ee1-1333-4e43-b4fb-ee8671883fc7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.339083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f508239a-6b7a-46dd-ac00-c19c562188e9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.363645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2f8dae3-dc07-4a78-ba73-272b2494c411]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.392737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d73b34d-109e-4dc0-90a1-a2ddf8d28eb0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 148801, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.415206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41ec47d0-d915-41a2-a849-b9c5210e0425]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.438256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa735c58-490a-495d-8cfc-e654d610a05c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.460376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68a34bd9-3977-47c9-8189-0b2947e2b9a6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.488619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db3ef612-33b7-4509-895c-21a3240920b6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.489198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:48.489474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:48.489737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:41:48.490011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:41:48.505207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:41:48.505611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df5be4f2-66f1-4f83-9134-5e0ddfa4ab86]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.510218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0d2e7f5-2c09-4ba3-a206-1a6c57758c5f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.511318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:48.545408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:48.546870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:48.552001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:48.552408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:48.552804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:48.576375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e4880d4-b3b4-43ef-8b39-df29f017671a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.620669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[df325e78-c9a8-4ec8-8876-7ad131b9e65f]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:41:48.620669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:41:48.620669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:41:48.620669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:41:48.620669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:41:48.620669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:41:48.620669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:41:48.620669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:41:48.620669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:41:48.620669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:41:48.620669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:41:48.620669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:41:48.620669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:48.622111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df325e78-c9a8-4ec8-8876-7ad131b9e65f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:48.622255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:48.623793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:41:48.626507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ffa77e0-462c-4fc6-9be9-eb49e25ef98e]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.626507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:48.626507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:48.626507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:41:48.626507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:48.627201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3b6d316-c848-41d7-8ae1-bc11d986d7ea]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.628475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f96095d-3f0b-4114-b08d-565c7a93f94a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.629803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f50d294d-4c20-4b47-8d02-77ca6e2c0015]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.651371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65fb78b1-aa43-4ab4-8ba4-efd6ee81efee]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 340465, 'reachable_time': 36792, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 148819, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.658244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:41:48.658502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23391a0d-09c7-4a52-a141-b62aaeb44aa3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:48.659626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:11Z", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:41:48.661970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 604e940a-6986-426c-8b0d-3019a27f37be. Wait time elapsed: 2.531 Jul 01 09:41:48.676751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 604e940a-6986-426c-8b0d-3019a27f37be. Wait time elapsed: 2.532 Jul 01 09:41:48.676751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:41:49.347479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.686s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:41:49.348041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:41:49.349143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:41:49.349538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:49.350064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:49.352038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37884573-7333-42eb-b0d3-f70cd367e527]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.395342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7dce59e-8706-4e99-a471-51637637d84f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.432907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7e41bc6-2f19-4f7a-8aae-adc8662d5f62]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.464331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b97f9d0-e979-45db-ba01-aaebb4489872]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 148822, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.487661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90566d25-1567-4eb9-b2cd-71ef1d7f92f6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.507518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05918486-55ee-4148-8737-0b6a1262d89d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.529727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ab5c65f-2700-4f2f-9092-3e4c564c9dff]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.551318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71e51c3b-2788-4cc8-bc05-0d3094ca3dfb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.551976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:49.552268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:49.552901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:41:49.552901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2842d7f3-dab5-478a-9591-7ca6f311a57a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:41:49.569700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:41:49.570144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08ddaf7b-4b7e-4bf2-a453-c33d0692d81c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.573251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[521b8358-898b-46c0-8f22-c55de3f0d83f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.574633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2842d7f3-da, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:49.619263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2842d7f3-da, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:49.619984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:49.620376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:49.620766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2842d7f3-da, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:49.621294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2842d7f3-da, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2842d7f3-dab5-478a-9591-7ca6f311a57a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:99:f0:6c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:49.665408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e917cce8-a07f-41b7-a02b-db6461023434]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.700364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[03ca8c18-1a6e-47af-86eb-99e552f0636f]: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:41:49.700364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:41:49.700364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:41:49.700364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:41:49.700364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:41:49.700364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:41:49.700364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:41:49.700364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:41:49.700364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:41:49.700364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:41:49.700364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:41:49.700364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:41:49.700364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:49.701369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03ca8c18-1a6e-47af-86eb-99e552f0636f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:49.701485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:49.702875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:41:49.704632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78aaa327-e19a-4d46-bc45-dc9dac9cc562]: (4, ['qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.704632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:49.704632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:49.704857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:41:49.705032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:49.706192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[feabe439-4a2b-45ad-b9d2-f3e39c499c66]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.707148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31baae9b-292d-417b-9428-bebacad71c85]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.708496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6033360-dbff-4a88-96b7-9ebe47562977]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.729541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5023c61b-32cd-409a-8da3-a7f5ed54e583]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 340572, 'reachable_time': 30504, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 148841, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.741974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:41:49.742246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71239b57-7964-4984-92ff-52d33e24865b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 203e4741-69b9-468f-97b4-76a7d63b11b8: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2842d7f3-da not found in namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8. Jul 01 09:41:49.743276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1569289415", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184" Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:08Z", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:13Z", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "95238a63-f36c-4bb6-8f99-a580b7ab2a66", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6e:e1:05", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 1996, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.184", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:09Z", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:36:16Z", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2842d7f3-dab5-478a-9591-7ca6f311a57a", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2bcb40f2-7b62-4be1-960b-d15ec2950831", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:99:f0:6c", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "203e4741-69b9-468f-97b4-76a7d63b11b8", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2013, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:36:12Z", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:37:11Z", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "cb56461a-8202-4d02-8bd0-f8f6f22907b4", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:41:49.745746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 604e940a-6986-426c-8b0d-3019a27f37be. Wait time elapsed: 3.615 Jul 01 09:41:49.755206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 2, update_id 604e940a-6986-426c-8b0d-3019a27f37be. Wait time elapsed: 3.615 Jul 01 09:41:49.755206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:41:49.940432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-041cae3b-d94a-4fd6-9b45-82b881b219bc tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['203e4741-69b9-468f-97b4-76a7d63b11b8'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:41:51.002058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-71f802ba-b137-48fc-b467-f700b6eb0def tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got router deleted notification for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:41:51.146166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.401s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:41:51.147274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 203e4741-69b9-468f-97b4-76a7d63b11b8 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:41:51.148894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:41:51.149240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:51.149525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:51.150766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29ae2959-12cf-4674-a4c1-8922780256ad]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.169022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79c611dc-5368-44a2-a10e-324ea05cbcb6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.193730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f844397c-38fb-43cb-ad89-e948f6bf60af]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.226703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ea60d19-6b74-48f7-b56a-f5643e1bb565]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 148848, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.249750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ad2e15b-c98a-4a34-a8a2-467ed60f843c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.270551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[feb74eda-7a91-4977-b2e9-a481934d6ee1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.305714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfe017e4-a207-4b95-9178-c5f1cab70ac1]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.326683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32581f6a-bfd4-452e-abef-d2f5e7dc7f79]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.327606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:51.327949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:51.328384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:41:51.367971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64cfc25d-6898-4268-a90e-59186a88a44c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 340748, 'reachable_time': 32936, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 148854, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.372276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.004s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:51.372672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:51.376017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:41:51.376498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" acquired by "process_external" :: waited 0.003s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:41:51.414026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00a839ea-9a63-4d83-93e9-9112f943a22f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 340748, 'reachable_time': 32936, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 148855, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.416072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:41:51.416541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:41:51.440398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65d54f52-0d28-4b81-88f9-760a0bd1da33]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.623064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ccc4428f-b847-4db7-a367-603291713c9c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.685219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9284c59b-d407-472f-adc0-789980fdd85b]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.878423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[878de467-8169-4bdc-a941-c54af55936c2]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.879009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:41:51.879620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:41:51.880140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:41:51.880515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" released by "process_external" :: held 0.504s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:41:51.881521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:41:51.881863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:41:51.882339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:41:51.882845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:41:51.908390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[747d9cff-b68a-4d40-93de-4fd0c552f45d]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:51 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:41:51 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:51 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:41:51 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:51 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:41:51 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:51 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:41:51 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.942098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[216115c0-78fe-487d-a2e3-b099c10e45fa]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:51.956062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57e1c860-6d13-4669-8c58-a1156e469582]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:51 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:41:51 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:51 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:41:51 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:51 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:41:51 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:51 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 09:41:51 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.006084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9aee0c5d-ec46-48a6-8564-fbc1c3c75a1f]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.006699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:41:52.006984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:41:52.007307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:41:52.007946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" released by "process_address_scope" :: held 0.126s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:41:52.008381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:41:52.009044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:41:52.009658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:41:52.010073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:41:52.010578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:41:52.035944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[947dce01-3272-41e7-af95-8723cf92d290]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:52 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:41:52 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:52 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 09:41:52 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:52 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 09:41:52 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 09:41:52 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:41:52 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.086222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d919097-e754-4fe4-a8e3-cf5d4affaf3c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.099282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63b3624e-9355-4e27-bc33-ba0b0ea494ef]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:52 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 09:41:52 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:52 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 09:41:52 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:52 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 09:41:52 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 09:41:52 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 09:41:52 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.154306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[274c4290-25b2-477a-ba74-fcd599eb53da]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.155071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:41:52.155407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:41:52.155824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:41:52.156116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8" released by "apply_metadata_nat_rules" :: held 0.147s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:41:52.156638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-203e4741-69b9-468f-97b4-76a7d63b11b8 Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.haproxy Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 203e4741-69b9-468f-97b4-76a7d63b11b8 Jul 01 09:41:52.157233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 09:41:52.158779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/203e4741-69b9-468f-97b4-76a7d63b11b8.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 09:41:52.176642 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-203e4741-69b9-468f-97b4-76a7d63b11b8[148899]: Proxy listener started. Jul 01 09:41:52.180603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 203e4741-69b9-468f-97b4-76a7d63b11b8 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 09:41:52.181147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, update_id 604e940a-6986-426c-8b0d-3019a27f37be. Time elapsed: 2.437 Jul 01 09:41:52.181512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 1, update_id 7b9bf97a-85bd-4a44-9d00-0fb50240f027. Wait time elapsed: 2.241 Jul 01 09:41:52.181815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 203e4741-69b9-468f-97b4-76a7d63b11b8, action 3, priority 1, update_id 7b9bf97a-85bd-4a44-9d00-0fb50240f027. Wait time elapsed: 2.241 Jul 01 09:41:52.182334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:41:52.221338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.039s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:41:52.221728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 203e4741-69b9-468f-97b4-76a7d63b11b8 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 09:41:52.223095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:41:52.248457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60f4e048-3ae7-4329-87ae-eb6bbf8fd5fd]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.250207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a130b02-7bae-4809-a086-43dc1e4dde65]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.251523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f55242c8-8477-477f-af3a-d95847ed28cb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.252184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 09:41:52.253020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69e23227-b71f-4868-8307-f6fefca1372a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.276760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65118065-d5bc-4eb1-9c07-c12516b32516]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 340748, 'reachable_time': 32936, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 148905, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.277692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:52.278306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:52.300513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8efe0b86-75c4-4ae9-ab97-7eeb7e77fe45]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 340748, 'reachable_time': 32936, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 148906, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.301686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 09:41:52.302494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:52.302909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/203e4741-69b9-468f-97b4-76a7d63b11b8.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:52.303162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:41:52.304185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 203e4741-69b9-468f-97b4-76a7d63b11b8 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 09:41:52.305719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4718c793-e484-45fb-8fa4-31b7c324c54c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.328993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f465b28-e200-434e-90b2-ad24200328bf]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 340748, 'reachable_time': 32936, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 148907, 'error': None, 'target': 'qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.340759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-203e4741-69b9-468f-97b4-76a7d63b11b8 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:41:52.341259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f5ddc70-86aa-4fa9-8c4d-8f3ba29d09d7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:52.341904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 203e4741-69b9-468f-97b4-76a7d63b11b8 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 09:41:52.342414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (203e4741-69b9-468f-97b4-76a7d63b11b8), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:41:52.343008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:52.343486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:52.343950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:52.344384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:52.344862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 203e4741-69b9-468f-97b4-76a7d63b11b8, update_id 7b9bf97a-85bd-4a44-9d00-0fb50240f027. Time elapsed: 0.163 Jul 01 09:41:53.503878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ae8bea26-3b9d-4cdf-8b9c-2e41d61281b8 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['13856d84-bd5d-4874-910c-6123a2d422b7'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:41:53.504893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 1, update_id f1be2267-a889-43ae-b3ce-e32113a6681c. Wait time elapsed: 0.001 Jul 01 09:41:53.505143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 1, update_id f1be2267-a889-43ae-b3ce-e32113a6681c. Wait time elapsed: 0.001 Jul 01 09:41:53.505541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:41:56.010910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.505s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:41:56.012074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:41:56.013374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:41:56.013697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:56.014173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:56.015631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04cc16d5-fdfd-4fb1-841b-991187ee04ab]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.035149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eaec8f11-9f7e-4e47-855c-b58da3ec112e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.060188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dcf4e440-9927-44e2-9209-b5478bdc83ea]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.097340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9cdf6ab-7f17-4e95-a6d6-de85aa4775e4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 148943, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.119017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38e90407-18b3-4622-b4b8-6fd3c09d1158]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.141578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bb541b6-20e7-4b33-a129-fa1a4c8cd376]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.162196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e01317dc-d53e-440c-a32f-ca654975b911]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.183052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6493cb54-48eb-42d4-814a-a77b0c275a12]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.184342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:56.184814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:56.185200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:41:56.185578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:41:56.202242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:41:56.202536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54a6c524-e53b-466d-8f24-a386f9c185a1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.206261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f0a0756-6b22-441d-8692-84a616a30b68]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.207777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:56.208321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:41:56.209182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:56.210001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:56.210474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:56.210936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:56.211421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:56.264660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-14ee7a4e-8ba6-4944-b03c-747fb024ea82 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['13856d84-bd5d-4874-910c-6123a2d422b7'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:41:56.270249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6af047ec-4b8a-4dcb-8f90-c05b61d88e0d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.303144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[87a6e448-55fd-4c1c-b6c4-e91320881b5b]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:41:56.303144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:41:56.303144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:41:56.303144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:41:56.303144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:41:56.303144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:41:56.303144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:41:56.303144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:41:56.303144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:41:56.303144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:41:56.303144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:41:56.303144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:41:56.303144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:56.304417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87a6e448-55fd-4c1c-b6c4-e91320881b5b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:56.305846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:56.308190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:41:56.312406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97e298fb-c775-4e9e-87b5-dee548b9bffc]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.313262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:56.313667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:56.313925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:41:56.314360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:56.315656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37c0349d-43a2-4922-8b1d-3faceb4081ef]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.316634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e689ab28-87ec-4962-b183-52158fde2e8c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.317755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc250fdf-ca43-4ff6-8a33-bbfa74b78888]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.339043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1d51ff2-0f37-4ed7-9972-7acf45104e6d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 341236, 'reachable_time': 44520, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 148961, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.342005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:41:56.342294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c73bf2d-7a24-4976-ac07-798ca18a0561]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:56.343325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 1, update_id f1382297-d6ff-4637-a879-36adab3043c7. Wait time elapsed: 0.079 Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 1, update_id f1382297-d6ff-4637-a879-36adab3043c7. Wait time elapsed: 0.080 Jul 01 09:41:56.345944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:41:57.931570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.586s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:41:57.932218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:41:57.934287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:41:57.934842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:57.935190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:57.937063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d55cf8e-abb2-45cb-86ee-cefba27a488b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:57.966189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[149952d7-79f8-42ae-87e3-2ae7cf54c49f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.044287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[385268be-84c6-4429-b55d-3307583130e5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.082316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04b57aa9-e301-44a4-bce0-a8ee5c1c6062]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149004, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.157013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[facb771d-3952-4a0b-83cb-f4fd47f49f5c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.185079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef1d99bc-1da7-46a3-904a-932f88c77910]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.221772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6748079a-3907-4e4f-a24f-70bd624789b1]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.269112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88b48e11-edbe-4b1c-bd85-17f50317e502]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.269967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:41:58.270316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:41:58.270654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:41:58.271004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:41:58.296084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:41:58.296493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7cda54e9-ea14-438d-8247-67ef690d8d77]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47365747-ea1d-49a9-8d3b-521ee9a5988c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.301043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:58.332258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:58.334505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:58.335523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:58.336105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:58.336700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:41:58.388966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bc15091-3304-4bfc-a742-25815b56f974]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.433316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[dfba2fef-f466-43d0-9e04-92dc95c09284]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:41:58.433316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:41:58.433316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:41:58.433316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:41:58.433316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:41:58.433316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:41:58.433316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:41:58.433316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:41:58.433316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:41:58.433316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:41:58.433316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:41:58.433316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:41:58.433316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:58.438514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfba2fef-f466-43d0-9e04-92dc95c09284]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:58.439834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:58.443325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:41:58.446863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[055a2b7b-d000-46fd-808c-e5a54621c3ea]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.448014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:58.448427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:58.448745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:41:58.450207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:41:58.451377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0beb8d4c-c5d8-4842-a2f7-229f3bdf1982]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.452861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a85ac3b4-72b7-4cbc-9938-4fe98e057974]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.453896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac3c1d68-7df5-4523-99d9-02d9b2087eb7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.477267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e855497-731e-4f3c-bd7b-1c928ab9f986]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 341434, 'reachable_time': 26552, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149035, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.485021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:41:58.485405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b41508b-19ac-459a-89df-514d2dce28bd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:41:58.486710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:57Z", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:41:58.490215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:41:58.505772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1be2267-a889-43ae-b3ce-e32113a6681c. Wait time elapsed: 4.988 Jul 01 09:41:58.505772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1be2267-a889-43ae-b3ce-e32113a6681c. Wait time elapsed: 4.988 Jul 01 09:41:58.505772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:00.815774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.323s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:00.816333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:00.817915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:00.817915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:00.818437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:00.820263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5462402-5133-4f91-ba9f-b2b23e6e1257]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:00.834873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b630f37c-7c11-44a8-991a-8c8f64c98d61]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:00.858308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ec0b17d-89a1-4951-bd4d-4b0d25e7912a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:00.891854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69cf1668-8de0-4d8d-9d10-cf3b3146c9cd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149046, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:00.920257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c11bef3-d1c3-46cb-b869-15d846b5034e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:00.941700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e597d428-fc4a-4b98-806a-5318bd972ed2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:00.971475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8356e9e2-f45f-4f71-b5d4-c26ff25af405]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:00.993233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[695d85db-714e-4feb-9168-aaa214bbb746]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:00.994076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:00.994474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:00.994823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:00.995174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:01.010414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:01.010722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5daabed-14a6-4c23-8bbb-faa229ec865a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:01.014517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82be58b7-ef1f-457f-9d22-13476792f75e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:01.015581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:01.060240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:01.061093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:01.061640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:01.062278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:01.063521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:01.119269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[edf4cd63-7eb8-4539-8df0-bbc0f4ebbcab]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:01.153977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ffd57caf-40ca-4ebb-8065-9f88751d9266]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:01.153977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:01.153977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:01.153977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:01.153977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:01.153977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:01.153977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:01.153977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:01.153977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:01.153977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:01.153977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:01.153977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:01.153977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:01.154848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffd57caf-40ca-4ebb-8065-9f88751d9266]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:01.155169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:01.157980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:01.160679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[febf8d18-1a86-4d6a-b5ce-31ca59cd034c]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:01.161439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:01.161768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:01.162025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:01.162368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:01.163404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8708ebc-dbcb-4076-b81d-0c397fb7c796]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:01.164386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10ed0f1e-9436-4934-96fb-b189e802360f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:01.166070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35bb909f-0ad8-406d-adac-df46283cb350]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:01.188113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41a05baa-535d-4f2b-b86b-cb5488c19399]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 341714, 'reachable_time': 29184, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149064, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:01.194151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:01.194493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[732dbf66-2585-4485-9040-351aa1cf2443]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:01.195486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:57Z", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:01.198242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1382297-d6ff-4637-a879-36adab3043c7. Wait time elapsed: 4.932 Jul 01 09:42:01.211593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1382297-d6ff-4637-a879-36adab3043c7. Wait time elapsed: 4.932 Jul 01 09:42:01.211593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:02.018714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.821s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:02.019386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:02.020188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:02.020499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:02.020776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:02.022325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa8d0f46-77c7-4a73-95cc-77fd5d9c81b2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.039439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af9cbd80-1d1c-4687-a1df-93913339c0a8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.064087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b62df74f-b5dd-4c71-ab67-a0304a34a046]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.104898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ab6b078-9ea9-41ad-9688-c6b8f756d654]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149067, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.127400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba7c62fa-d390-44db-b724-0e11f12b568b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.150175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[664571e0-336e-433a-bc3f-6d10ee522eaf]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.172601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59d08c0f-b916-4a7a-bac2-bdb73e958e5f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.194291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4165320a-bc4c-493a-8048-e42d325b175e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.195039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:02.195562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:02.196119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:02.196529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:02.213585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:02.214111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c589306-e281-41a5-a343-f5a1a4785b89]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.217858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23400575-bd0b-48f0-8c62-4afd0024cca9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.218986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:02.265693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:02.269213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:02.269605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:02.269995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:02.270564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:02.328294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83554193-23e9-42f2-bc74-b02d4422a428]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.382691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b1c4badd-f5b6-480c-89eb-6be4235798fe]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:02.382691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:02.382691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:02.382691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:02.382691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:02.382691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:02.382691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:02.382691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:02.382691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:02.382691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:02.382691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:02.382691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:02.382691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:02.383604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1c4badd-f5b6-480c-89eb-6be4235798fe]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:02.385003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:02.386599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:02.388583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f76547b0-dd97-482d-9203-138d94fe95df]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.388583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:02.388583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:02.392705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:02.393706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:02.394865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af485df3-9c6c-4877-914b-7b3503fce0a1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.396062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd53e3b2-96f0-437f-9a8e-4545af9181e8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.397087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27a02775-5551-46ad-b283-61f6a9078d2f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.431981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a41a8cbf-c88b-4d42-b656-0197d0f9fa9f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 341836, 'reachable_time': 25852, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149087, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.439443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:02.439776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b9a5e6a-d85d-4e09-8b69-71f863a6fefe]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:02.443216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:00Z", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:02.445703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1be2267-a889-43ae-b3ce-e32113a6681c. Wait time elapsed: 8.940 Jul 01 09:42:02.463512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1be2267-a889-43ae-b3ce-e32113a6681c. Wait time elapsed: 8.940 Jul 01 09:42:02.463512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:05.054786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.610s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:05.055397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:05.056558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:05.056888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:05.057173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:05.059238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f03091d-2ceb-43ad-a636-40de086d4fe9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.082333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30cee503-7963-4a09-9546-a3e4c4698173]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.122362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae9888b8-32f8-41d1-a1fc-0f4b2771b824]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.173660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4075353a-98f3-4788-bbf2-521ecbb31f73]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149136, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.200942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66482455-c753-4d5c-977d-a7760a4cfaf8]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.223942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b75324c4-44ee-4204-8d5f-bb34f6a038ac]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.247108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[687606bf-7ea5-4ce6-b226-54da59c6eb9c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.281012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5caae92f-12e5-4b4f-a0c6-0420512fde2c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.282202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:05.282686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:05.283154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:05.283660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:05.311315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:05.311912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eab83668-a819-42f1-aa4a-90cf5072069a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.315842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68bff1e9-be1a-4326-8bdb-97faca5e0fcb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.316942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:05.348398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:05.349067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:05.349464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:05.349844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:05.350229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:05.390449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a529cd9-e46a-4048-adb5-8dbe61ac5e53]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.420280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7109c9cd-7076-47a5-a5a1-7689b25a309a]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:05.420280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:05.420280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:05.420280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:05.420280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:05.420280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:05.420280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:05.420280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:05.420280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:05.420280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:05.420280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:05.420280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:05.420280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:05.421119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7109c9cd-7076-47a5-a5a1-7689b25a309a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:05.422117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:05.423515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:05.425285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39458f77-89a0-4cf8-acdc-f8982cc41cdd]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.425285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:05.425285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:05.425519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:05.425733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:05.427885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0396dcb-2fc9-4e7e-827b-d733a6ae704b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.429099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d5c3e15-05a2-460e-ab7c-13f61d3b7d75]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.430799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ca9bd0a-2e7e-4c11-9ff7-91543e75e40f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.455405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d366510c-0fc9-41a2-9a5c-71fff63143ed]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 342143, 'reachable_time': 39872, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149160, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.459012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:05.459503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49507f78-f93c-4f06-b447-6c1094945a1e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:05.461674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:03Z", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:05.466102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:05.479402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1382297-d6ff-4637-a879-36adab3043c7. Wait time elapsed: 9.210 Jul 01 09:42:05.479402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1382297-d6ff-4637-a879-36adab3043c7. Wait time elapsed: 9.210 Jul 01 09:42:05.479402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:06.192846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.717s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:06.193496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:06.194030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:06.194349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:06.194627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:06.195857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[935055cd-cb90-42b0-9def-239cde50477c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.209102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd3dd683-a34e-40cc-8f92-59d84c538546]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.230741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e673a47-d13a-44f9-b9d4-01a803966f78]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.257985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65fed697-a49e-4337-85ed-68900d8a157e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149163, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.278818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f988cef-1152-4c97-b46e-a5e7d1b02064]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.298718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[228c66f6-7e25-403a-8937-bbf7e6e7b0a8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.318271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00c2a0bd-61aa-4510-a712-8735298994a6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.329111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[966406c8-1609-4065-8af8-dc96215920e4]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.329720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:06.330069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:06.330384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:06.330737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:06.346232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:06.346739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7a03781-8f72-42fc-bffa-706075c31f1e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.349538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48b7d267-2900-4ff7-b738-28f7e5f94888]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.350525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:06.381961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:06.383170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:06.383815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:06.384409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:06.385040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:06.430295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da2eadc6-9b75-4a41-b6db-6c402a5bfadd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.469386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5fbb95e0-e629-41b5-863c-d080af4460f5]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:06.469386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:06.469386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:06.469386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:06.469386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:06.469386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:06.469386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:06.469386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:06.469386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:06.469386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:06.469386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:06.469386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:06.469386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:06.470970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fbb95e0-e629-41b5-863c-d080af4460f5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:06.472477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:06.474456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:06.476354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cad27c76-f98b-487d-b39f-16936a855308]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.476354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:06.476354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:06.476354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:06.476354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:06.477460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18c8c235-31c7-44b2-91e7-799d9dc6e961]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.478387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf8d12b4-749f-4643-989e-7d2a5e3929c2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.479355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab19d7b9-78d4-419e-ae63-63e5ee62f210]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.499875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[853be8b7-af00-48e3-b91a-ae92dbbf860f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 342252, 'reachable_time': 15116, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149181, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.507942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:06.508219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58aef8d1-96bc-4093-8516-8dcd81b0a110]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:06.509291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:04Z", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:06.514305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:06.535408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1be2267-a889-43ae-b3ce-e32113a6681c. Wait time elapsed: 13.012 Jul 01 09:42:06.535408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1be2267-a889-43ae-b3ce-e32113a6681c. Wait time elapsed: 13.012 Jul 01 09:42:06.535408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:09.344259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.827s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:09.344833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:09.345959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:09.346291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:09.346581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:09.348109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84835103-d1ea-48c8-9b30-78405f58063f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.361334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38d3514e-404a-4d8e-9b08-b7937031e7e3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.382299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d10b98c5-b63c-450b-8950-5400cc352789]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.415021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bba66b6-9e4c-4cf1-9ddd-e576e4ac3f5a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149266, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.438043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb665367-4e05-4c2b-beda-a4c45fe11a0b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.458012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf992c41-59c6-4a9e-a821-4892c7d3be9a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.478529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dadadfc3-ef84-45bf-bdce-847319702e4b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.498346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28989801-7575-4ee4-9cbd-69c4f1a2c5fa]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.498897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:09.499226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:09.499533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:09.499839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:09.517027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:09.517310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94828e80-6217-489a-8c30-7100709ac399]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.520006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[826b2c41-6171-44cf-951d-c389aeec4015]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.520930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:09.562005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:09.562686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:09.563089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:09.565610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:09.566017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:09.603756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e805404-b4b6-4522-ba39-b6a71e894e6f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.636215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8225219e-d609-470a-8eec-a2dcc627bbe0]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:09.636215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:09.636215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:09.636215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:09.636215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:09.636215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:09.636215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:09.636215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:09.636215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:09.636215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:09.636215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:09.636215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:09.636215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:09.637884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8225219e-d609-470a-8eec-a2dcc627bbe0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:09.638355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:09.640026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:09.641785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9915f96f-2fbb-46da-a26b-50ea853ea28f]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.641785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:09.641785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:09.642012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:09.642280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:09.643644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43d96575-8565-4cd5-a2b3-64c59c9b3ef9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.644651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[005139a9-1858-4958-a0e9-826239ac9372]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.645731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19296851-ddf6-4430-a726-8d5790828cff]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.668008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d67db56e-4cee-4301-9ac5-c7c1e8458055]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 342567, 'reachable_time': 35580, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149284, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.672972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:09.672972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0679b01e-3703-479d-a61b-f6e5927d4ab8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:09.673957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:07Z", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:09.676430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1382297-d6ff-4637-a879-36adab3043c7. Wait time elapsed: 13.410 Jul 01 09:42:09.688132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1382297-d6ff-4637-a879-36adab3043c7. Wait time elapsed: 13.410 Jul 01 09:42:09.688132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:10.485168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.810s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:10.486128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:10.487477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:10.489096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:10.489610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:10.491321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b117d92d-c529-45e5-bf6e-81815257579e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.511420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d8dce37-2f06-4292-b068-891e9e724f56]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.533299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb1711a7-d77d-411e-8fde-c9a8ded9971d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.561432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb46a9c7-609b-4e15-9350-1f267958aa06]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149288, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.583018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f66a428a-81c4-4215-9a1f-5c9ef635ce1b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.603513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bc84be3-9453-447d-b52e-0e5f4d3217da]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.623752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f54b383-64ee-4f6c-af49-d62ffe6dafca]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.644141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[767a6cf4-3f94-4185-a936-782fdb65d8df]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.644774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:10.645114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:10.645446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:10.645786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:10.660514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:10.661406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d387f1b-9cec-4c4a-bf24-9e404af6c59f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.664058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d70ec5b-57be-475a-be9e-bd14254f4bc1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.665021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:10.693296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:10.694315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:10.695111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:10.695576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:10.695967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:10.738908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe50ab49-1084-452b-af22-0201e299097a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.765994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e9defab4-24e5-4197-a524-6804cbeb1f9e]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:10.765994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:10.765994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:10.765994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:10.765994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:10.765994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:10.765994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:10.765994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:10.765994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:10.765994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:10.765994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:10.765994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:10.765994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:10.767023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9defab4-24e5-4197-a524-6804cbeb1f9e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:10.767866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:10.769276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:10.771926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94f322b1-1158-480e-8e35-93edc6249bb0]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.772170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:10.772495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:10.772747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:10.773083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:10.774070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de9d38fa-5f1c-4755-8b3b-fc76396d103c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.775071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9987c9f7-3f19-4050-b8e9-6585dec15959]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.776098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f5ed6d0-d102-4e68-bbd9-3900b107e1a1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.796504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5f07344-dc4c-444c-bd2e-2aa9855483ae]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 342682, 'reachable_time': 36052, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149306, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.801837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:10.802301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[911c3357-9cb8-4e35-b557-2243e32e41cb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:10.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:09Z", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:10.806763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1be2267-a889-43ae-b3ce-e32113a6681c. Wait time elapsed: 17.300 Jul 01 09:42:10.818538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1be2267-a889-43ae-b3ce-e32113a6681c. Wait time elapsed: 17.301 Jul 01 09:42:10.818538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:13.693561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.888s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:13.694158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:13.694718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:13.695083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:13.695379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:13.696539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2762cba-e1b3-465c-afdb-4e5baae8c95e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.709485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86e5030d-4a0d-4fcd-a1d2-a0aca7bbdb8f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.730733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51cd8be9-23cb-4231-b2d0-f022c5f142ef]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.760094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbc546fb-7967-4612-9b70-7560d6f4da89]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149356, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d185068b-bef6-4098-bbe1-bdbd98c62ba8]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.811260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e29978b-757f-4b3f-8aca-8126916b2120]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.837413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24a55266-5f75-413b-8aae-6d371e8c004d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.858353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0bc2ba57-5351-428e-b29a-9b52d940933a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.859044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:13.859486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:13.859813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:13.860286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:13.875984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:13.876526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1818ee5-1b47-400f-b29b-38a042c9935b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.879414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82624f9c-31c9-42ce-865b-f0b7f05f641b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.880511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:13.915341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:13.915341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:13.915341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:13.915341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:13.915341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:13.950021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52ee422c-412e-47ec-a639-881c6f782e79]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.983280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[60feaacf-bc1a-4eb9-bf9d-d5db1a646efe]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:13.983280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:13.983280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:13.983280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:13.983280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:13.983280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:13.983280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:13.983280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:13.983280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:13.983280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:13.983280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:13.983280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:13.983280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:13.984168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60feaacf-bc1a-4eb9-bf9d-d5db1a646efe]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:13.984684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:13.986224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:13.989140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97d4c033-2908-4c45-b966-df625898f18f]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.989140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:13.989140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:13.989472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:13.989902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:13.991146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbb3e547-30f6-4847-bf9f-0f5412a291c2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.992629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03538aaf-9c42-47ae-a688-2b6325e4348a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:13.994307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fe3c629-7b37-49d6-974c-6473c2239590]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:14.013964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8cb86607-6dff-4307-acad-e5ce8f41393a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 343002, 'reachable_time': 27192, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149374, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[431e5346-a1f8-4985-b566-a19a2fbd03b6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:14.018694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:11Z", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1382297-d6ff-4637-a879-36adab3043c7. Wait time elapsed: 17.753 Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1382297-d6ff-4637-a879-36adab3043c7. Wait time elapsed: 17.754 Jul 01 09:42:14.023270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:14.730230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.711s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:14.731408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:14.731759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:14.732085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:14.732366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:14.733618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d649edd-9081-446f-b3e7-0d19112658a9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:14.748819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa8a8dc8-4c89-4b5d-859d-248f4a6d194d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:14.769570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc7ef32a-3c5d-4deb-915a-be74e27d8f50]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:14.800463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20c62b14-d623-4235-b3c5-d3cd4586b487]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149377, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:14.822036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43979680-c551-4cc6-9547-a158dc7885f1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:14.843784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb901d16-1e56-433b-a806-fd83320f9a4e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:14.864427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f65726c-2362-4566-a04f-61a0554fd1a6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:14.887561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc5039da-c82b-4c81-9ca4-173683edc45e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:14.888177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:14.888454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:14.888720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:14.888996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:14.906043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:14.906706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6090b308-b015-4747-8350-06b3afe8120a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:14.909980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9de58632-70d0-43dd-a988-273878cdac8d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:14.911492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:14.941571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:14.943825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:14.944443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:14.945011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:14.945551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:14.994882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a87ef7a-10c8-448f-b5db-80408b614e9a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:15.031743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[095d8e6a-1e73-4466-8268-a4d71ac1560f]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:15.031743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:15.031743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:15.031743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:15.031743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:15.031743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:15.031743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:15.031743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:15.031743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:15.031743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:15.031743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:15.031743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:15.031743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:15.032637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[095d8e6a-1e73-4466-8268-a4d71ac1560f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:15.033323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:15.034718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:15.037507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de4daa70-98a1-482c-9044-662621e0446f]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:15.037507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:15.037507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:15.037507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:15.037507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:15.039057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b38bc154-1d35-425b-b455-06b0c8ba28d5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:15.039430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73419e15-823d-46f1-93e0-39599593daba]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:15.040462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3aeace62-28d2-41c6-84b4-382e054d701d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:15.062031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1a561e0-677e-4fa9-a72c-7111e15d31b8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 343106, 'reachable_time': 33484, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149395, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:15.070660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:15.071025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c4f4f8d-fcc1-4e01-b37f-be85daebb261]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:15.072246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:14Z", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 13, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:15.076189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1be2267-a889-43ae-b3ce-e32113a6681c. Wait time elapsed: 21.571 Jul 01 09:42:15.088443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1be2267-a889-43ae-b3ce-e32113a6681c. Wait time elapsed: 21.571 Jul 01 09:42:15.088443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:15.089426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:15.089762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:15.090937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:15.091265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:15.135084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:42:16.898556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-634be8a1-409c-4543-8f0f-78ea31a70d22 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['13856d84-bd5d-4874-910c-6123a2d422b7'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:42:17.868706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.793s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:17.868973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:17.870407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:17.870755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:17.871036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:17.872193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[738c1078-a163-4a6b-82c6-ee20bbe50552]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:17.886277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4346a717-d8a9-48c1-bbba-fbbc4c43951b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:17.907096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46d1a219-c6f5-4f08-85e0-46c6344feae8]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:17.937230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2fc0cd7-7298-4b7e-b641-459eb0f42fa4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149403, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:17.959747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6641f7b0-81bd-4696-8075-94f5c2853ca6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:17.981988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[adc781f7-5da1-4141-9d95-1f58beb75248]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:18.002232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bba9937-d124-431a-869f-c068957c0f30]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:18.023566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc67169a-ef49-45e3-bb06-faa25d439b56]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:18.024233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:18.024571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:18.024896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:18.025252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:18.040691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:18.041446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a60eecaa-5b2a-4e3b-9904-79de2af77b75]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:18.046129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a18cd19f-6ece-4366-b647-b870b2ccb916]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:18.047139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:18.079671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:18.080651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:18.081756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:18.082764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:18.083323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:18.147920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb711f9b-ca7e-411a-97b8-720a08aba67a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:18.181640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bb29bca4-b8e6-4688-a568-c52b498917e4]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:18.181640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:18.181640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:18.181640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:18.181640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:18.181640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:18.181640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:18.181640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:18.181640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:18.181640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:18.181640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:18.181640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:18.181640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:18.182569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb29bca4-b8e6-4688-a568-c52b498917e4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:18.182902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:18.185364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:18.187842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[813a92e0-e972-4a4c-9f49-1604eb25c2dc]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:18.187842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:18.187842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:18.187842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:18.187842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:18.189081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4df597e2-1699-4195-9cf4-dba32893c816]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:18.190577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f56c91b-2f2c-437a-982f-de30aa695633]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:18.191780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2b44a62-cde6-4da9-8e76-5cb27aa4e82c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:18.216546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33638454-f0a1-4193-a499-c5c8a65b54d9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 343420, 'reachable_time': 39852, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149425, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:18.228166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:18.228464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddd5a8e1-1ebd-4408-9749-41ba0f35d6e2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:18.229570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fe32bb6-a95e-43ad-90b4-f6cade43faf2", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.155", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "8c5ae86f-ac0a-44eb-9236-ce73e4a43c90", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2381, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:1e:ac:8e", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a8ba51de-ed0b-4cf4-b6e0-561f55be4d48", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:42:16Z", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:16Z", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:18.232911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:16Z", Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3 Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 1, update_id eb05ddd8-83d1-48f5-896a-a3fe943b6911. Wait time elapsed: 1.335 Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 1, update_id eb05ddd8-83d1-48f5-896a-a3fe943b6911. Wait time elapsed: 1.335 Jul 01 09:42:18.256497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:19.220345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.986s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:19.220951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:19.221549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:19.222056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:19.222339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:19.223509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b226a7f-f230-4f2c-abd9-3ce82c772f39]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.238903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3460ae92-e052-4b9c-865f-86483a36898d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.260978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83c902e3-c5b7-4b6b-9068-ad72399da2a9]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.290109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c2b3134-964f-4ab9-83e7-8ed9f37f1d75]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149469, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.311238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bdf7c7a-daf9-4a2a-9a9d-efa184894dd6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.332517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67b1e729-75d7-48ad-8927-9ae4dec5a7fd]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.353187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72b09103-813a-43d3-b6c3-23dfba12e7cd]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.374424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c17ccef-748e-48cc-b9c9-b62e93979492]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.375072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:19.375374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:19.375677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:19.375960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:19.390132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:19.390455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[221a8e8e-81bb-4617-8f51-d2bd5a90a6e5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.393119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f017d16c-27c3-4293-afc5-08751b21424b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.394321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:19.426385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:19.427223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:19.427713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:19.428313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:19.428912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:19.468674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c32a9b5f-0413-4f31-bcca-5b901f87f781]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[57543122-5077-419c-93a3-9447d46e6ac9]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:19.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:19.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:19.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:19.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:19.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:19.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:19.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:19.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:19.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:19.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:19.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:19.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:19.499633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57543122-5077-419c-93a3-9447d46e6ac9]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:19.499869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:19.501383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:19.504007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1596f6b1-d9d3-42d9-be12-97f97676b5ff]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.504007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:19.504007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:19.504007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:19.504007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:19.504710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f492968-1802-4606-9061-9ab1c4c03b75]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.505704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb38723c-b5da-42e2-9f5b-eae1f72f22a6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.506770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ad895b3-2160-4517-bb8c-686b688732de]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.524898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8258d1ac-018f-449f-b9e3-0b5f3e852750]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 343555, 'reachable_time': 17304, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149487, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.528673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:19.529090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f7eaa03-ce82-4286-afa1-9dfad58660b5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:19.531099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fe32bb6-a95e-43ad-90b4-f6cade43faf2", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.155", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "8c5ae86f-ac0a-44eb-9236-ce73e4a43c90", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2381, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:1e:ac:8e", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a8ba51de-ed0b-4cf4-b6e0-561f55be4d48", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:42:16Z", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:16Z", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:19.534782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:18Z", Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1382297-d6ff-4637-a879-36adab3043c7. Wait time elapsed: 23.270 Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id f1382297-d6ff-4637-a879-36adab3043c7. Wait time elapsed: 23.270 Jul 01 09:42:19.544611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:21.832212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.296s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:21.832766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:21.833503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:21.833825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:21.834099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:21.835255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aecba4fe-ef1a-4cd8-846a-fe22914f88e9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:21.849630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9126ee16-5c7b-4f6b-9f71-3adad0d98cd9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:21.870245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78fcc6c7-d427-4897-afde-3b147b79caeb]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:21.897480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[760d920b-5b2a-486b-9507-fa43c867b1c6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149493, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:21.918566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[542fad03-16c3-4b02-906a-00a6955a2732]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:21.938157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d57d0bb5-1e29-48b9-91ae-0a119a8b4665]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:21.957840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d18b4671-f448-4d59-b149-8f20d15d1e74]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:21.977510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5a1b8dc-5e47-4e04-9ce9-8dba6961a0c6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:21.978066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:21.978348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:21.978620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:21.978897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:21.993965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:21.994361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e986b67-7b31-41d2-8bf8-4cf554525b6a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:21.997677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d8a94ee-c54e-4c3f-8128-f24eb5fae0c8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:21.998698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:22.025327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:22.026335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:22.026862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:22.027621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:22.027958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:22.067442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8fda3257-149b-4f8c-9eef-5d25329231bc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:22.096874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[aa37cf31-a945-47aa-81a9-4716edfdaaf3]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:22.096874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:22.096874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:22.096874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:22.096874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:22.096874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:22.096874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:22.096874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:22.096874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:22.096874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:22.096874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:22.096874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:22.096874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:22.097776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa37cf31-a945-47aa-81a9-4716edfdaaf3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:22.098138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:22.099766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:22.101621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a724f84a-9d4f-45e2-8791-9f3555a43995]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:22.101621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:22.101788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:22.101982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:22.102480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:22.103629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9dcae011-8e5b-4c9a-aec3-9684a0d70544]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:22.104524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d0f43ea-48d6-4897-91ea-a251886acf3d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:22.106184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4bf3d288-b075-49d3-8d8c-36876eb49653]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:22.124872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e80aea3-a1f0-466e-bfcf-ca237be67553]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 343816, 'reachable_time': 20720, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149511, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:22.128911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:22.128911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ebd0c06-aa0a-4721-b92b-626def1421df]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:22.129501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fe32bb6-a95e-43ad-90b4-f6cade43faf2", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.155", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "8c5ae86f-ac0a-44eb-9236-ce73e4a43c90", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2381, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:1e:ac:8e", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a8ba51de-ed0b-4cf4-b6e0-561f55be4d48", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:42:16Z", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:16Z", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:22.132343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:20Z", Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3 Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id eb05ddd8-83d1-48f5-896a-a3fe943b6911. Wait time elapsed: 5.231 Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id eb05ddd8-83d1-48f5-896a-a3fe943b6911. Wait time elapsed: 5.232 Jul 01 09:42:22.144532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:22.878149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.747s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:22.878711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:22.879564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:22.879886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:22.880164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:22.881350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eca7bd0c-01b2-4421-bbac-417aea6fa294]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:22.898420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[340b6389-464c-4f9a-a957-66ae36af9854]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:22.922341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa2c5df5-740f-4ff0-9556-202f3d9136de]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:22.950519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f635b5b4-dbda-495a-a502-9ba0b149f9f9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149514, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:22.970461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d81a6504-fd83-4bbd-b6f5-c0d1de64bffd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:22.991581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90bd1972-9f45-46b7-8bb5-df5e8f2f8b2b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:23.013423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13e0817a-b084-41da-b158-cf0170da9a55]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:23.033451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c55e2770-e9c9-4b16-a701-5b3a2d710f6a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:23.034052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:23.034392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:23.034697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:23.034973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:23.055682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:23.055974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6fc1b8de-1b5b-4cfe-a165-c25f4193e33b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:23.059779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2508b50-80e2-4bbd-ba73-427fd04b2d50]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:23.060875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:23.094009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:23.094816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:23.095824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:23.096312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:23.096848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:23.139594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[420046e7-eca9-4693-b156-7211b01eb400]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:23.176051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c81ee9af-31f8-4023-ab53-08ee18ad8b5e]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:23.176051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:23.176051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:23.176051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:23.176051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:23.176051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:23.176051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:23.176051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:23.176051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:23.176051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:23.176051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:23.176051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:23.176051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:23.177009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c81ee9af-31f8-4023-ab53-08ee18ad8b5e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:23.177243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:23.178829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:23.180670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[065a240e-c64b-402a-8f6c-cf2f30d368ed]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:23.181189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:23.181679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:23.182092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:23.182565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:23.185230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad65b965-5bb1-4410-8039-c1ad4893f691]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:23.186386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1744c320-9e38-4a5e-83ae-f6fcbd9f18f8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:23.188530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b50f34c3-1459-4b3a-b49b-274845ea1e3d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:23.213001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4afd0095-0400-46d2-8521-2ee2f453066d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 343921, 'reachable_time': 40024, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149536, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:23.220448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:23.220985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9a3784b-6e8c-4f38-9b7a-619fcf6a6aba]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:23.222359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fe32bb6-a95e-43ad-90b4-f6cade43faf2", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.155", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "8c5ae86f-ac0a-44eb-9236-ce73e4a43c90", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2381, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:1e:ac:8e", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a8ba51de-ed0b-4cf4-b6e0-561f55be4d48", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:42:16Z", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:16Z", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:23.225421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:21Z", Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id eb05ddd8-83d1-48f5-896a-a3fe943b6911. Wait time elapsed: 6.327 Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id eb05ddd8-83d1-48f5-896a-a3fe943b6911. Wait time elapsed: 6.329 Jul 01 09:42:23.241680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:25.851938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.619s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:25.852562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:25.853238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:25.853650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:25.853985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:25.855852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08f345c7-38fc-4147-b7a3-3b97e2fa5f38]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:25.876358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc113cc6-fbcf-4f81-b36c-f1b4172019d1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:25.900839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae61fff1-803f-4ed0-a5ee-4753bd23c781]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:25.929363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43fe2593-0943-4376-a076-a253fde4818b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149586, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:25.950779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ed9f42c-321b-4762-b638-b1af9b780536]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:25.974054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5525a093-f6a7-47b2-954c-0674dc338b00]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:25.994907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab7898e5-4511-4079-b17b-331e051fa27f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:26.016504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae996ac5-02b2-4b34-bda8-d4a0be7225fa]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:26.017082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:26.017430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:26.017694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:26.017972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:26.040640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:26.040906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06f6443e-eeeb-4cc3-a697-b63fd5cec774]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:26.045007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89c00c0b-6d8a-4878-abab-d7ac6253be56]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:26.046970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:26.077202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:26.077979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:26.078433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:26.079038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:26.079535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:26.129227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71eec271-af4a-42ef-bc0e-dc89f7227502]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:26.160943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[dbba86b4-2220-4317-a4b3-96ed8846da69]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:26.160943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:26.160943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:26.160943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:26.160943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:26.160943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:26.160943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:26.160943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:26.160943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:26.160943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:26.160943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:26.160943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:26.160943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:26.161993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbba86b4-2220-4317-a4b3-96ed8846da69]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:26.162165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:26.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:26.166195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[802434b6-bdaa-476b-b5d6-028d2619b234]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:26.166502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:26.166868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:26.167123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:26.168309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:26.169563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[227edb7a-6119-4f7c-8a58-bfd717a5c139]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:26.170718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[867e6832-bbb5-4248-bee0-e99f5563f0ed]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:26.171626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3358d66f-ada1-4b5f-862a-e6b6f60fd322]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:26.191504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7294f7a-1b61-431e-87d7-89ecd43887c0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 344219, 'reachable_time': 15564, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149604, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:26.197359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:26.198201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ad34e6d-497e-4f80-83ab-71885e508c57]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:26.199434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fe32bb6-a95e-43ad-90b4-f6cade43faf2", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.155", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "8c5ae86f-ac0a-44eb-9236-ce73e4a43c90", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2381, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:1e:ac:8e", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a8ba51de-ed0b-4cf4-b6e0-561f55be4d48", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:42:16Z", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:16Z", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:26.202309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:23Z", Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 22, Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id eb05ddd8-83d1-48f5-896a-a3fe943b6911. Wait time elapsed: 9.302 Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id eb05ddd8-83d1-48f5-896a-a3fe943b6911. Wait time elapsed: 9.302 Jul 01 09:42:26.215228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:27.007998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.807s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:27.008932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:27.010484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:27.010812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:27.011173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:27.012754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68ffbdb1-03a6-4e36-8274-1e1322572b47]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.024925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c58a9a6-822c-49eb-8e75-2b0525e94a52]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.046915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed31c2d2-20b7-4e6a-a82f-4c9329ce1598]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.076104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c6f3f4c-7ab6-4730-ab6f-36a5f1877f6a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149607, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.096349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03f2e751-bfc3-4a1a-8706-f8b2d1d2f8c6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.116817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a5a87a2-8871-4add-9c6b-be7047bd7547]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.137658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71861e7b-d9af-4a27-b368-14ce4eb5999b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.158577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9778ab3-54d4-4765-af7a-b0bc82894c06]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.159118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:27.159504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:27.159827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:27.160106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:27.175311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:27.175645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bedda3d5-5add-4693-971c-5025a00fb695]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.178451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ff37564-4a05-4dea-adbb-93869ebf99cd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:27.221682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:27.222657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:27.223122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:27.223616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:27.224142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:27.262338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ce3523f-0612-45d9-8b35-2bdb7eaf94ad]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.291721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a3187455-ed19-455d-8bc5-1559ed094927]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:27.291721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:27.291721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:27.291721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:27.291721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:27.291721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:27.291721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:27.291721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:27.291721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:27.291721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:27.291721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:27.291721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:27.291721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:27.292814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3187455-ed19-455d-8bc5-1559ed094927]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:27.293637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:27.295393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:27.297658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2cf03d0-a747-404f-994a-79df1263061a]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.297658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:27.297862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:27.297944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:27.298263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:27.299229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7fbe9709-73c9-4c6d-ab9c-a8f5d472da84]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.300182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[421263cc-2f34-49ce-b3ab-06ee4b0cf2fe]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.301042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bcdac3e-875a-4b47-a2c2-a01c70947c5d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.324113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ae3aa94-5615-4e82-b600-16a734556a45]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 344334, 'reachable_time': 35612, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149628, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.330152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:27.330519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da7326e5-6faf-4748-ba48-a48f9dba18aa]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:27.331706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fe32bb6-a95e-43ad-90b4-f6cade43faf2", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.155", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "8c5ae86f-ac0a-44eb-9236-ce73e4a43c90", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2381, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:1e:ac:8e", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a8ba51de-ed0b-4cf4-b6e0-561f55be4d48", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:42:16Z", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:16Z", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:27.334496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:26Z", Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 25, Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id eb05ddd8-83d1-48f5-896a-a3fe943b6911. Wait time elapsed: 10.436 Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id eb05ddd8-83d1-48f5-896a-a3fe943b6911. Wait time elapsed: 10.436 Jul 01 09:42:27.348283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:30.414588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.079s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:30.415767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:30.417247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:30.417878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:30.418241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:30.419587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a54b8739-902d-4671-9ef9-88ab9e012c8f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.440314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[695922f8-6fa0-4af1-94e2-b52083a89ffe]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.461580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40503bfb-facc-49e5-91ed-a4c4529461ea]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.489536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e89d183-15fd-4c33-8458-31d1ba475ae7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149679, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.514377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fabcaf1-0203-434c-899c-c1944c570c6b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.534287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b5e6091-0488-4798-8b9c-14f1b0695934]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.554035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b10193c-0893-4595-887a-72465414797f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.575501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9abe8033-4584-4835-955f-7fd2861147ba]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.576640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:30.576911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:30.577173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:30.577447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:30.592240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:30.592521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55937421-1623-4aea-9e0f-4cc515e47932]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.595247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de6bcd48-cec8-4e1f-aa6f-2f9827fd0952]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.596219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:30.625420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:30.627090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:30.627661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:30.628050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:30.628482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:30.667186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6bde8ccf-9535-4a05-b87e-f2e76e9a2373]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.693787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cb626ffe-fb1a-4783-b669-5c965bb337f5]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:30.693787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:30.693787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:30.693787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:30.693787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:30.693787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:30.693787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:30.693787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:30.693787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:30.693787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:30.693787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:30.693787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:30.693787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:30.694653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb626ffe-fb1a-4783-b669-5c965bb337f5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:30.695375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:30.696831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:30.699728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f17ee0b7-e662-43e1-a64e-9324c6285cbf]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.699933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:30.700406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:30.700663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:30.701003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:30.702063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0a56529-7462-4425-8499-b7e5b7717a38]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.703066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb5e9863-269d-4888-bf86-a10d09f6b03c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.704883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa7dbbdd-0beb-4f20-ba3c-4d64b49a69de]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.723824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0d4fe05-d4e8-4b7b-ae2b-4ef3d5169dc5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 344675, 'reachable_time': 27360, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149697, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.729938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:30.730299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[532496d0-f597-4c54-994e-1424722b91a1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:30.731403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fe32bb6-a95e-43ad-90b4-f6cade43faf2", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.155", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "8c5ae86f-ac0a-44eb-9236-ce73e4a43c90", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2381, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:1e:ac:8e", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a8ba51de-ed0b-4cf4-b6e0-561f55be4d48", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:42:16Z", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:16Z", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:30.733983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:27Z", Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 26, Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id eb05ddd8-83d1-48f5-896a-a3fe943b6911. Wait time elapsed: 13.835 Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id eb05ddd8-83d1-48f5-896a-a3fe943b6911. Wait time elapsed: 13.835 Jul 01 09:42:30.746519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:42:31.489879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.756s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:42:31.490494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:42:31.491714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:42:31.492010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:31.492296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:31.493674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5abfdb9-2c1f-4a87-bdda-cb9324d61b5f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.507509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a891075-7737-42b3-9072-63520b030af8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.528301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75acad11-d8ca-465d-952f-70789618be81]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.553961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75ba6a4e-4a73-49cf-b874-50eeabfa787e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 149700, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.575142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d146e63f-8aea-4c30-9304-cd032d2ca7fe]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.595667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3abc4500-a6ba-4ac0-b53e-2be39b2236f7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.615211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fdb0a15-7bd3-4db4-96d2-bbf0fe680dd0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.635669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44f1cd4e-43ee-47c0-89aa-f4e4b56fbe2f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.636916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:42:31.637254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:42:31.637525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:42:31.637988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:42:31.651983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:42:31.652304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e35909c-a64e-4984-923b-4558cbab5348]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.654964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[faf4df71-ed29-4275-9cd7-c04bf61f4bc6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.656012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:31.689309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:31.690259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:31.690655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:31.691199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:31.691620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:42:31.734391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[749bed07-a85c-4bbd-ac88-dcfce0717f55]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.764937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bc906d8a-4576-4412-a876-934f6dab0504]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:42:31.764937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:42:31.764937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:42:31.764937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:42:31.764937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:42:31.764937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:42:31.764937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:42:31.764937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:42:31.764937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:42:31.764937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:42:31.764937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:42:31.764937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:42:31.764937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:31.765846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc906d8a-4576-4412-a876-934f6dab0504]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:31.766161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:31.767712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:31.769497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c25de31-25dd-4674-b905-633d7d312d44]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.769497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:31.769497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:31.769731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:42:31.770425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:42:31.771084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79db41d5-a965-4c4d-825b-2a9caf88c61c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.772278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c36bfa4e-0c5c-4ac8-b09b-89b018f8f6d8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.773232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12ee2af6-c566-4754-85a6-1d534478d830]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.797147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57b8fbee-ca9a-47fe-bf5d-7fa2a02b376d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 344781, 'reachable_time': 15360, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 149719, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.799128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:42:31.799613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e262ad0-d8c8-4676-bac8-5900797fed57]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:42:31.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fe32bb6-a95e-43ad-90b4-f6cade43faf2", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.155", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "8c5ae86f-ac0a-44eb-9236-ce73e4a43c90", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2381, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:1e:ac:8e", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a8ba51de-ed0b-4cf4-b6e0-561f55be4d48", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:42:16Z", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:16Z", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:42:31.805499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:30Z", Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:42:31.817366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3 Jul 01 09:42:55.136798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:43:15.089898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:43:15.090525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:43:15.090525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:43:15.090667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:43:35.138603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:44:15.090600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:44:15.091259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:44:15.091259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:44:15.091494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:44:15.140208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:44:55.142416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:45:15.091614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:45:15.092298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:45:15.092298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:45:15.092586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:45:35.144300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:46:15.094522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:46:15.095163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:46:15.095163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:46:15.095320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:46:15.146263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:46:55.148475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:47:15.095619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:15.096723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:15.097161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:15.097447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:17.971653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c1e466fe-2652-4007-88c5-a1506cff784f tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['13856d84-bd5d-4874-910c-6123a2d422b7'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:47:17.972721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 1, update_id 03ce5988-9013-42f7-8dca-a4395707fb5b. Wait time elapsed: 0.001 Jul 01 09:47:17.972973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 1, update_id 03ce5988-9013-42f7-8dca-a4395707fb5b. Wait time elapsed: 0.001 Jul 01 09:47:17.973267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:20.229370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.256s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:20.230009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:20.230862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:20.231406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:20.231776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:20.233345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2abff09c-19a9-4f84-9937-e698adcdd43f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.247224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[421c89bf-28e3-4285-9728-d6b241e44041]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.268071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8f2fe2e-f85b-44e2-a9a4-4dc44110b0ed]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.299883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8dfa5d56-7035-4d87-bc4b-127e31280ddd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154322, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.320350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4523023d-9ab3-49f3-b3b8-ceb962482195]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.341074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba9ecc3b-b451-442c-9cf7-d7c550e72d97]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.361899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab169140-3180-4da6-a3ef-920760b610fe]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.382543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10440967-698a-4e39-95d1-62516dfc8aa3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.383189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:20.383572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:20.383909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:20.384255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:20.402168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:20.402439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1eeb2d6a-539a-4751-9b14-ff4949c91f3e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.405397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f2e64c5-8207-4493-a043-fcda281b2c45]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.406408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:20.441122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:20.441843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:20.442449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:20.442969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:20.444470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:20.491177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da43cf80-bfff-4624-84ac-9f8ef51a8580]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3dbeb379-f9f5-4ba7-8f3c-6cbca47b2039]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:20.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:20.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:20.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:20.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:20.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:20.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:20.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:20.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:20.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:20.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:20.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:20.522523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:20.523579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3dbeb379-f9f5-4ba7-8f3c-6cbca47b2039]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:20.524534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:20.526211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:20.528732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cba976dc-13f3-4dac-9961-e6c91165e6ff]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.529433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:20.529826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:20.530084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:20.530492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:20.532212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12385eb0-5fd3-40fa-a1ea-d12e23d338e6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.533360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b03dc26c-fa58-4c86-9aea-0bb4436f5fc5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.534256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0b41456-7047-4cd4-b203-bc9db8765eb5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.553978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9534eb03-c263-4c7c-bd8e-540336f6a5c6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 373656, 'reachable_time': 38252, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154341, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.556021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:20.556574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[621afe77-9a1e-49e4-a6f9-de3b28c22a7d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:20.558952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:33Z", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 31, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:20.563012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id 03ce5988-9013-42f7-8dca-a4395707fb5b. Wait time elapsed: 2.588 Jul 01 09:47:20.588682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id 03ce5988-9013-42f7-8dca-a4395707fb5b. Wait time elapsed: 2.588 Jul 01 09:47:20.588682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:21.294704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.734s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:21.295682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:21.296714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:21.297232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:21.297577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:21.298744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58bb31e0-3017-4a62-b393-93f2a18e2877]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.320691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03c9849e-087b-42a7-98c6-53e3b8942abb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.342875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c56bbf8-1228-4f53-887e-fd00d251fed0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.376854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4de508e6-9f06-49a0-91ed-06681631ffe2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154345, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.398676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2bf8e14-b37d-47f6-af8b-0f9eb81af70f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.419028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec55d58e-ce1c-48b3-b99f-8ea0c42de090]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.448616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c3d9b65-a80c-450f-99c1-e80957c50d80]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.470309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44f5a306-5a78-4828-bd62-f7b3c66cf875]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.471084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:21.471511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:21.471877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:21.472281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(50c1f514-8a76-40f8-ae90-25d1aa7ed4f6) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:21.489713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:21.490484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c14e90bd-dc6a-4b22-8495-f2a774d8367f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.493827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b3bd368-f3bb-4766-ac7d-9cd004bc577c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.494794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-50c1f514-8a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:21.542859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-50c1f514-8a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:21.543455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:21.544620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:21.545189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-50c1f514-8a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:21.545568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-50c1f514-8a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '50c1f514-8a76-40f8-ae90-25d1aa7ed4f6', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7e:44:ea'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:21.588825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a37fc7a6-5522-439a-9e2f-099d1a4fcabb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.625837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[599025a9-3ce7-4b96-be6d-705a814c5f5d]: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:21.625837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:21.625837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:21.625837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:21.625837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:21.625837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:21.625837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:21.625837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:21.625837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:21.625837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:21.625837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:21.625837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:21.625837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:21.627521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[599025a9-3ce7-4b96-be6d-705a814c5f5d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:21.628629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:21.631841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:21.633915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffff0976-6737-462f-8b55-b8431ddbf065]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.634042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:21.634439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:21.634890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:21.635389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:21.637262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a43a4133-584a-4262-8fbd-0eedf5a02d61]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.638389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66885205-3597-46ff-a54e-11c52520026a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.639342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f0563da-c565-46f5-885e-e7d70a551f04]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.663952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0d2af01-c7b9-4a47-ba05-16b4884cef41]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 373764, 'reachable_time': 16624, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154364, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.670006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:21.670360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf66f999-0df0-47fc-a618-e1010fd5398b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-50c1f514-8a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:21.671443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "50c1f514-8a76-40f8-ae90-25d1aa7ed4f6", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "01346cb2-684d-4de2-ba4b-411e5f7a71b7", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7e:44:ea", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2354, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:54Z", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:42:33Z", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 31, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a703f100-772a-4c37-b5ff-13c3b1dd9e58", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:21.675343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:21.688877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id 03ce5988-9013-42f7-8dca-a4395707fb5b. Wait time elapsed: 3.715 Jul 01 09:47:21.688877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 2, update_id 03ce5988-9013-42f7-8dca-a4395707fb5b. Wait time elapsed: 3.715 Jul 01 09:47:21.688877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:22.610118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-9cdfea43-98cc-434e-9506-b8e5c97b3632 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['13856d84-bd5d-4874-910c-6123a2d422b7'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:47:23.883770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ea896a98-2048-4d7f-a035-b43632f6e70d tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got router deleted notification for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:47:24.159061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.472s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:24.159466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 13856d84-bd5d-4874-910c-6123a2d422b7 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:24.160735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (13856d84-bd5d-4874-910c-6123a2d422b7), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:24.161184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:24.161708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:24.164298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2502f88a-6e8d-4672-8037-e6233df326ac]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.174936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8079313a-e789-4b28-aa9e-31e1bb64aa4e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.195948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d335489-74d3-4991-a072-3ff18bbdd059]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.226075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d66f259-6fd3-45d2-bb05-d89a8379a01c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154413, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.250185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2ce9725-2da3-49a4-add7-e1e52eaaf6a3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.273701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20aa41a2-bb0c-4013-bae9-daa3c71286bf]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.295780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[297a3d5e-499f-4649-94b3-f36302980ee3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.321568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9382f184-7f28-4ead-8a29-5d9f55131640]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.322886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:24.325429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:24.325926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:24.351445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3285a34b-fe1c-4959-9cf7-6c951b373b60]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 374048, 'reachable_time': 37300, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154419, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.352817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:24.353162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:24.353666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-13856d84-bd5d-4874-910c-6123a2d422b7" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:47:24.354040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-13856d84-bd5d-4874-910c-6123a2d422b7" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:47:24.354578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:24.354911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:24.355270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'f468294c-2aa2-4e74-a8cc-4b8c0197c6fd', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:ae:b8:cd', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '13856d84-bd5d-4874-910c-6123a2d422b7', 'device_owner': 'network:router_gateway', 'standard_attr_id': 2350, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.216', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:41:52Z', 'updated_at': '2022-07-01T09:41:55Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-f468294c-2a), ns(qrouter-13856d84-bd5d-4874-910c-6123a2d422b7) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:47:24.372423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-f468294c-2a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:24.373140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54b2ea37-a05f-4871-bfff-cd83e670f6d5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.377842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4fb7975-4e74-4e12-b20f-5f8208fa4e69]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.378987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-f468294c-2a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:24.379410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:47:24.381877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-f468294c-2a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:24.382798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-f468294c-2a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:24.383203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-f468294c-2a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:24.383627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-f468294c-2a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:24.384003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-f468294c-2a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f468294c-2aa2-4e74-a8cc-4b8c0197c6fd', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ae:b8:cd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:24.440481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2be577f3-1d50-4043-8adf-6d76ef923b46]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.490816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bfced096-b4c3-4a15-a0c6-aee1f51e04aa]: Network interface qg-f468294c-2a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:24.490816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:24.490816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:24.490816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:24.490816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:24.490816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:24.490816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:24.490816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:24.490816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:24.490816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:24.490816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:24.490816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:24.490816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f468294c-2a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:24.491912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfced096-b4c3-4a15-a0c6-aee1f51e04aa]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-f468294c-2a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.492314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-13856d84-bd5d-4874-910c-6123a2d422b7" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:47:24.493694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-13856d84-bd5d-4874-910c-6123a2d422b7" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:47:24.523710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8e89bee-e896-4343-a952-0c025c712a14]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.707535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38b6f35c-ab90-4381-8b65-020227ac1222]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.725186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fdaefde-44bf-4dd6-a12e-bbc9e3303f14]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.898909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ac51a8c-1618-44a3-b435-43c0ebb3e358]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.899579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:47:24.899904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-13856d84-bd5d-4874-910c-6123a2d422b7" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:47:24.900546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-13856d84-bd5d-4874-910c-6123a2d422b7" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:47:24.900546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-13856d84-bd5d-4874-910c-6123a2d422b7" released by "process_external" :: held 0.547s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-f468294c-2a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f468294c-2a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f468294c-2a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:24.901561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f468294c-2a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f468294c-2a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:24.903802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:24.906243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c5be352-974d-4150-aad9-c82cd6ab1dec]: (4, ['qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.906243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:24.906243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:24.906243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 13856d84-bd5d-4874-910c-6123a2d422b7 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:24.906243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/13856d84-bd5d-4874-910c-6123a2d422b7.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:24.906604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e57db71-c6af-46dd-8e92-2949fba55a19]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.907294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee3eece6-c78e-4584-ba26-31521e75a0cf]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.910083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2be17cd0-d757-4652-9195-0d7914218513]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.931447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d88d2234-79d6-4c1d-b8ca-d68a4eb796dd]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 374048, 'reachable_time': 37300, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154435, 'error': None, 'target': 'qrouter-13856d84-bd5d-4874-910c-6123a2d422b7', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.934548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:24.934826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d87f96af-bd34-4c20-896c-a7f0975b9295]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 13856d84-bd5d-4874-910c-6123a2d422b7: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f468294c-2a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f468294c-2a not found in namespace qrouter-13856d84-bd5d-4874-910c-6123a2d422b7. Jul 01 09:47:24.935949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-792711358", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216" Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:51Z", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:21Z", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f468294c-2aa2-4e74-a8cc-4b8c0197c6fd", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ae:b8:cd", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "13856d84-bd5d-4874-910c-6123a2d422b7", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2350, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.216", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:41:52Z", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:41:55Z", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 1, update_id 58dda11b-e628-4e18-b667-e7bd4bec9b11. Wait time elapsed: 2.326 Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 13856d84-bd5d-4874-910c-6123a2d422b7, action 3, priority 1, update_id 58dda11b-e628-4e18-b667-e7bd4bec9b11. Wait time elapsed: 2.327 Jul 01 09:47:24.939049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:24.972785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.035s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:24.973226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 13856d84-bd5d-4874-910c-6123a2d422b7 was not found. Performing router cleanup Jul 01 09:47:24.974963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3c3f1d9-c10f-49c5-8e95-96308348ea8c]: (4, ['qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:24.975816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:24.976153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:24.976479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 13856d84-bd5d-4874-910c-6123a2d422b7, update_id 58dda11b-e628-4e18-b667-e7bd4bec9b11. Time elapsed: 0.040 Jul 01 09:47:26.656263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c549175e-f00f-445e-aa54-136d97017e13 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['19b5d892-dd48-419b-8faf-b3c1e06f0f75'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:47:26.657898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 1, update_id 6ac9414d-6fff-4f7c-8d2a-574ceea26347. Wait time elapsed: 0.001 Jul 01 09:47:26.658208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 1, update_id 6ac9414d-6fff-4f7c-8d2a-574ceea26347. Wait time elapsed: 0.001 Jul 01 09:47:26.658565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:29.293105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-2f5b3134-9f0f-4634-8140-9495237f3cc0 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['19b5d892-dd48-419b-8faf-b3c1e06f0f75'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:47:29.875325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.217s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:29.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:29.876982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:29.877422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:29.877778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:29.879939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36936746-e37b-48e3-a2c0-202a70ad1daa]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:29.899033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a4eba51-2d90-4d9a-a657-87cecd9cc859]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:29.920461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b67709f-1bbc-49ba-b524-4b5d08adbb2d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:29.950527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e05f21ef-702d-42ae-b6eb-4a5d961c4ca2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154472, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:29.973086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc8e5cbe-e1ec-4271-85a8-74ebbd7b3fad]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:29.995892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[daec22b0-c980-4fdb-990f-d13f52aa4db8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:30.025656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12014f4d-e31e-460f-8b56-6179bd9e0868]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:30.047202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac21f2f6-6d3c-4c4a-8c72-814bd840fd23]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:30.047971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:30.048329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:30.048653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:30.048984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:30.066592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:30.066869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93470f7f-0e92-49ec-ad3d-70b83b3c9d6b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:30.070069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[364bcf14-460f-422d-a709-632c936fda2a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:30.071114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:30.071482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:47:30.072262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:30.073837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:30.074470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:30.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:30.075617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:30.118700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9097d81e-5fb3-45b8-90a0-566a2200c5a9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:30.176893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[121d13cf-88b9-44db-9726-fdb0a2918fac]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:30.176893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:30.176893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:30.176893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:30.176893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:30.176893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:30.176893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:30.176893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:30.176893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:30.176893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:30.176893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:30.176893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:30.176893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:30.178888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[121d13cf-88b9-44db-9726-fdb0a2918fac]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:30.180631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:30.185048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:30.190833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c03ba44-7673-4c02-83e7-9ad310e1b557]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:30.192022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:30.192566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:30.194132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:30.194785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:30.196321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43511b45-3c1a-44dd-936a-09e4e3d9f5e7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:30.199191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aded3cab-6abc-4970-ab27-522268a2946e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:30.201767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb0c3346-d710-4b9b-a7a2-ffa932870e01]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:30.256037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ae500ab-6572-4f0d-812f-546d428dc764]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 374621, 'reachable_time': 36824, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154497, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:30.259627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:30.260157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[302a6195-96d5-4cae-bd1d-3ca35ab1b7a6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:30.262127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:30.268188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:30.285521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 1, update_id a33bd4dc-b242-4ec4-82f5-aaf86f711a3c. Wait time elapsed: 0.986 Jul 01 09:47:30.285521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 1, update_id a33bd4dc-b242-4ec4-82f5-aaf86f711a3c. Wait time elapsed: 0.986 Jul 01 09:47:30.285521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:31.855833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.575s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:31.856450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:31.857006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:31.857330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:31.857610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:31.858882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16e4dc26-1876-49f0-a164-a7e4dfcb4028]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:31.877812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d110f2a-7b6d-4aa5-a9f0-7b770e0d8cc4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:31.898348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9fe02d43-87d5-4830-9abb-828a9ab872ad]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:31.924381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7089581d-b38d-4dc4-b2e0-7dfd7d48e05d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154548, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:31.944701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ffb0965-5b44-48ce-9ddf-68bbb446cce6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:31.965110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f814208-0884-4294-b2a5-be8ee5b9c5e2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:31.984875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34fa39fe-91da-4d95-8afb-b491e5b2a7e7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:32.004794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5911d2f7-6514-4a21-8b06-e304baecd7aa]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:32.005845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:32.006306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:32.006611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:32.006844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:32.030358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:32.030613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1742603c-834f-4853-bf8c-252817b47fe3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:32.038139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f63bc7d-73d7-46c5-8c93-8d218a452c47]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:32.039088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:32.073814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:32.074730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:32.075317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:32.075985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:32.077090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:32.122085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ae36207-79c6-46c5-8969-a2ee1a5d5493]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:32.161385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5ceff063-9d24-427f-b938-1430f095abcc]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:32.161385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:32.161385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:32.161385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:32.161385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:32.161385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:32.161385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:32.161385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:32.161385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:32.161385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:32.161385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:32.161385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:32.161385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:32.163741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ceff063-9d24-427f-b938-1430f095abcc]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:32.164668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:32.167497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:32.171293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbdb5c92-c3db-4e47-91e3-6e935a4b1d82]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:32.172283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:32.172700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:32.173072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:32.173536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:32.174673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95b664ca-1765-4d69-bf70-db8da6b27318]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:32.176073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4867d399-a1e2-4cd1-8a63-40d7ddd57e68]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:32.177965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99871609-1945-494e-a576-520bf2214a80]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:32.200007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7906e403-5798-437d-a57c-84e385f14981]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 374818, 'reachable_time': 38448, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154567, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3e48f34-1196-488b-a3cc-55fd0325db80]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:32.204977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:31Z", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id 6ac9414d-6fff-4f7c-8d2a-574ceea26347. Wait time elapsed: 5.548 Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id 6ac9414d-6fff-4f7c-8d2a-574ceea26347. Wait time elapsed: 5.548 Jul 01 09:47:32.210291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:34.665702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.460s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:34.666349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:34.666872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:34.667485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:34.667763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:34.668907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73c1f1c1-505d-4cb9-949b-7895a8fb7ef9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.684012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[644548f6-bfc2-4315-9c4b-ceeac84066b3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.711789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e609fd2-9dc4-4e9f-a4f8-e41fd81cfc26]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.748273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0908a49-0b0d-4638-88c4-8438905029f7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154616, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.770541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f68de4e-3214-4079-9b48-921c93177f7d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.794243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55292a8c-1130-4257-aeda-31577343ca6d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.816853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c369af8-93f4-4beb-a9cf-1e7db2f8e773]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.837448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a98e829c-58e2-4b9c-a887-8f37c64ca717]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.838152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:34.838430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:34.838698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:34.840143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:34.859990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:34.861315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d69e5db9-c0a1-4374-a0cb-4fd9c557fe93]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.865240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d0c35c9-198d-4e28-8cbe-da2ba9c34994]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.866540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:34.904915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:34.905991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:34.906534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:34.907179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:34.907870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:34.953984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[581a2935-a5f0-4c38-ab3e-654f36b0e86b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.985160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[58ce5d7a-cd54-441d-9642-36dcd077bf44]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:34.985160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:34.985160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:34.985160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:34.985160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:34.985160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:34.985160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:34.985160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:34.985160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:34.985160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:34.985160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:34.985160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:34.985160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:34.986256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58ce5d7a-cd54-441d-9642-36dcd077bf44]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:34.986348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:34.988270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:34.991456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a73ecf31-f99a-4b54-9df0-de7468693c2c]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.991456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:34.991456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:34.991456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:34.991456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:34.991891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47bff6ce-66a4-4b92-bd8f-40932df72c20]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.993039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f591fcc-e0bc-4216-bec2-a55d1446a6d9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:34.993934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b314c6c-a797-4b10-8420-8eb8e44fe5d8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:35.012508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b318771-37f5-431a-b977-fad72b33ad1a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 375101, 'reachable_time': 43980, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154634, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:35.014819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:35.015253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4da829a-e6a3-4b36-b1de-ed0e873e9016]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:35.018141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:31Z", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:35.022366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:35.032033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a33bd4dc-b242-4ec4-82f5-aaf86f711a3c. Wait time elapsed: 5.729 Jul 01 09:47:35.032033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a33bd4dc-b242-4ec4-82f5-aaf86f711a3c. Wait time elapsed: 5.729 Jul 01 09:47:35.032033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:35.150132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:47:36.001140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.978s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:36.002285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:36.003469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:36.003885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:36.004308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:36.006261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1cfe9c3-d53d-411e-8374-225d7eae43cb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.030690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c84b7f37-c4fb-425f-ac94-94f7b88cc25a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.056781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[805f4352-e3f0-4ea5-ace0-fafd02fc0406]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.122062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aed19d1c-8cfd-4c58-9168-22bf3039d062]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154641, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.150732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e99b270-c691-4651-8124-5c2a4aa614e9]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.176805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e19d4d3-919a-46e1-b44f-22b6ce6093c5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.215017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[317cf407-632b-446c-892c-6a5c29bfe385]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.249811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b73ab759-6b85-4d89-ba13-e1c4af870a2b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.251096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:36.251391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:36.252031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:36.252317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:36.273970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:36.274359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be0d38bc-43d5-44ac-92d6-f921e8f910b7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.279799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad91acf5-7b77-4096-aeb9-282ae63a5030]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.282291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:36.316973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:36.319602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:36.322721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:36.323203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:36.323682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:36.369194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fefa3e5-6ea1-4b9f-9d32-e782f84d84b4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.444875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e14d7fff-4881-4784-9a8a-4978be864389]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:36.444875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:36.444875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:36.444875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:36.444875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:36.444875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:36.444875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:36.444875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:36.444875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:36.444875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:36.444875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:36.444875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:36.444875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:36.446471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e14d7fff-4881-4784-9a8a-4978be864389]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:36.448376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:36.452318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:36.457127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd162b99-b478-4139-8b88-d1fb1fa7b2dc]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.460243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:36.460785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:36.461792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:36.462928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:36.464831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b6bc7a3-3938-4d28-ae1f-f51e12ccc3a1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.465827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[797cd414-df3b-41a0-8385-0daa00571d7f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.467070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c06fbbac-d48f-47aa-ae47-4901b04baaa7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.498219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a23d462-84b7-40f2-908d-3fea040d0892]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 375238, 'reachable_time': 44984, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154660, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.504376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:36.504671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c60cdf9e-ba81-4e13-9b63-139175a9d4ca]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:36.506309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:31Z", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:36.509000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:36.529442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id 6ac9414d-6fff-4f7c-8d2a-574ceea26347. Wait time elapsed: 9.852 Jul 01 09:47:36.529442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id 6ac9414d-6fff-4f7c-8d2a-574ceea26347. Wait time elapsed: 9.852 Jul 01 09:47:36.529442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:39.163941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.654s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:39.164534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:39.165142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:39.165472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:39.165754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:39.166910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da4d9c84-eaa2-41e6-828d-36d7ce9b696a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.204108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c35b4cdc-5aec-4bae-83cb-dc1ff7b840f9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.241056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9b5f8d4-11c5-4341-ab39-adc3bff269fa]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.317200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd724cca-7ff9-41d1-9805-e5d2b2707416]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154697, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.350494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a11c090-34b7-48fa-b0cb-11d4979b9476]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.383803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14f59ca9-27ec-44a8-ad75-b6d1f74d4358]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.405491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ac540c6-311d-468b-a369-9a9ed41fbadd]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.428904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0673c5c-1e1e-4b86-ab9b-739136ea0d43]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.429477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:39.429755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:39.430022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:39.430302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:39.447480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:39.447773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d4dd411-f59c-4734-879d-48e3764d481b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.450582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60280185-1765-48a6-ab4c-507d88423adb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.451853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:39.488326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:39.489369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:39.489876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:39.491687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:39.492173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:39.535790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[933af8b2-b7fb-4cab-896e-a445928e5cd2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.562673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2acd433b-7b7e-418b-811e-a7804088883d]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:39.562673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:39.562673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:39.562673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:39.562673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:39.562673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:39.562673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:39.562673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:39.562673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:39.562673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:39.562673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:39.562673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:39.562673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:39.563758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2acd433b-7b7e-418b-811e-a7804088883d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:39.563984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:39.565860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:39.569993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[877d2af2-e2f6-446f-9dea-2042fbf42f8f]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.569993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:39.569993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:39.569993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:39.569993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:39.569993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbab6d07-2868-422e-9642-0998033e04d9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.570690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84327c73-09c7-42b6-895e-f6206437f462]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.571085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36d6f743-6aa9-46da-8458-b206781f0065]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.589599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3a25cea-db58-481f-8328-79aa813c517c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 375558, 'reachable_time': 44176, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154731, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5f16d9c-571f-492b-937f-ea8636b33bfa]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:39.593620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:37Z", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:39.597066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a33bd4dc-b242-4ec4-82f5-aaf86f711a3c. Wait time elapsed: 10.301 Jul 01 09:47:39.616878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a33bd4dc-b242-4ec4-82f5-aaf86f711a3c. Wait time elapsed: 10.301 Jul 01 09:47:39.616878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:40.285357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.690s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:40.285964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:40.288121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:40.288539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:40.288879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:40.290093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d56f2e3-2ec4-4555-82ed-fbea75994b76]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.303078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[746eb73b-cf53-45ce-8ad7-4e0cf8fcc9ab]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.323426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4997170a-0d61-47cb-a118-c545195f2f72]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.350719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cac571e0-0dd0-4188-96a9-98257c201dc5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154735, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.371145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4931d13-dd30-4caa-8b06-a281361c71f0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.392154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb2f8b3a-e9fe-486d-94c1-790ec506c547]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.413443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75b632ff-3e2b-4362-8769-8e88d6f73336]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.436498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7f21dfd-fcad-4169-8ccc-fd8d497658e2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.437135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:40.437474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:40.437804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:40.438152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:40.454212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:40.455151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19214922-94ea-4afe-8d02-e2494a1f4deb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.458016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[231d2682-4946-41d0-8b75-807d602c8a3c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.459383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:40.502791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:40.503722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:40.504338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:40.504733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:40.505781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:40.548054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0703a70f-8b9a-4e2e-93a8-aaf9c50ffe8a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.582920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d617ce33-bcab-440e-b861-8df03e4725d0]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:40.582920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:40.582920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:40.582920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:40.582920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:40.582920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:40.582920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:40.582920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:40.582920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:40.582920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:40.582920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:40.582920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:40.582920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:40.584105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d617ce33-bcab-440e-b861-8df03e4725d0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:40.584659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:40.586603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:40.589240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41cf0bb5-f416-4955-9312-41bcdb4ff8df]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.589240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:40.589240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:40.589240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:40.589240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:40.590898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c34be590-dbc6-409b-b23f-4dab52e9093c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.591870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7221e18-d717-404b-b029-e852275e95f1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.593384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a61ba748-a75e-4c7d-9464-31221695e664]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.613486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61e9321d-d9fa-4aa0-98cd-4526906182e2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 375661, 'reachable_time': 26620, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154753, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.620235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:40.620495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8e0d88b-52d2-4d93-9c9d-9303d608a3e2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:40.621505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:39Z", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:40.624702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id 6ac9414d-6fff-4f7c-8d2a-574ceea26347. Wait time elapsed: 13.965 Jul 01 09:47:40.638663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id 6ac9414d-6fff-4f7c-8d2a-574ceea26347. Wait time elapsed: 13.965 Jul 01 09:47:40.638663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:43.392684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.770s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:43.393288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:43.393850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:43.394168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:43.394446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:43.396217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbdca99e-bd9b-4987-8824-53db74fff045]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.408801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99f04ceb-d84b-4427-b826-a14b21c8d4a1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.431242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4314a617-e3a2-4651-92a0-e23dac2433a7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.460922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6032122b-81f9-49e7-be95-4e05d1f187c9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154759, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.482087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c279ee98-4dcf-4b31-a95a-d4f4e920e5f7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.503412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[088fbf3e-5ca5-4d8f-ad61-b023c843a37a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.524325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbb95447-fae0-496f-acd3-671a7c1576b0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.544274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd08a1ba-16e2-433a-81d4-72fab5728fbd]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.544842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:43.545111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:43.545378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:43.545791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:43.561235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:43.561699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09fcb458-c1bf-4996-a183-7610e7d66980]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.564591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7eff8cc-3f3f-4ee0-a961-82772eb5ef20]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.565755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:43.604483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:43.605446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:43.605907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:43.606539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:43.607150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:43.643947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5260d312-aa05-449b-9f9c-ee30fb1610b4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.683537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5637b1f3-734f-4740-bac6-b775600d3ae8]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:43.683537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:43.683537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:43.683537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:43.683537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:43.683537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:43.683537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:43.683537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:43.683537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:43.683537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:43.683537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:43.683537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:43.683537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:43.684580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5637b1f3-734f-4740-bac6-b775600d3ae8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:43.685113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:43.687015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:43.689420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[592dffb3-c4ca-4ad6-b7bc-22236c83953b]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.689420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:43.689420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:43.689420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:43.689420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:43.690404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[450564b2-7633-41b1-b142-0b1955597a96]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.691320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad7687c8-5340-4ce7-b70c-a7dde8ccf5a0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.692278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acdc0659-99d6-48c9-9e11-7dc6570f86ee]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.715860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8028dba1-89a2-4fec-94fb-ab3ecd8b3867]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 375972, 'reachable_time': 38948, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154777, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46c4f8fe-ae32-4e79-b7a0-d47bf3e16de4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:43.720695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:40Z", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:43.725224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a33bd4dc-b242-4ec4-82f5-aaf86f711a3c. Wait time elapsed: 14.428 Jul 01 09:47:43.741597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a33bd4dc-b242-4ec4-82f5-aaf86f711a3c. Wait time elapsed: 14.428 Jul 01 09:47:43.741597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:44.422497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.700s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:44.423119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:44.424617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:44.425104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:44.425450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:44.428701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbbd9ae0-7b2c-4773-81ac-ffb4b303bcc3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.451756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55f8c674-e5d3-4195-bab3-bbae7de15d11]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.482021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3d70980-9035-4847-89bb-407796a8006c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.544753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c2ceb92-b3ae-46e8-b80c-6e148e728f16]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154789, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.586891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1cc15be0-0bb1-405b-88a4-901166fc5540]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.644403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ac281c4-cb18-4eeb-be88-b4b68ab321a4]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.694564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0704e8d7-1649-429c-a61f-9337526ace81]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.736954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecca9711-5264-4df4-95e1-1b129352d2d0]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.737829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:44.738253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:44.738664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:44.739102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:44.781558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:44.781811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a05ee43-b5ba-4188-ad9c-7d2e6ce55696]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.787687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76091c44-0a66-44d0-9c65-dc1d5cdc54cd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.788450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:44.816758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:44.817511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:44.818760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:44.824012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:44.824510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:44.890315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7acbcb39-ea15-4470-af74-de1183794ee4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.923902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3c2e6f52-9dd3-4cd7-b6b7-083515dcbe0a]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:44.923902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:44.923902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:44.923902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:44.923902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:44.923902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:44.923902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:44.923902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:44.923902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:44.923902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:44.923902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:44.923902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:44.923902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:44.924988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c2e6f52-9dd3-4cd7-b6b7-083515dcbe0a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:44.925259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:44.927224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:44.929733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba7e13c4-411e-4539-a3b1-e931c7f62c21]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.930065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:44.930448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:44.930710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:44.931118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:44.932276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[330286f7-77d9-4874-91a9-5d0315e8f411]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.933416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c47b4bb0-9ae3-4b6f-ac5a-545b55330957]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.934381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1f839cf-5a45-4db4-b05c-72ab636168eb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.954207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[354f00ec-1d6c-4ce9-ba8d-cf43cbd3da12]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 376080, 'reachable_time': 31440, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154845, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d11ffbb-c710-48b3-b93f-326199e237fd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:44.960275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:42Z", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id 6ac9414d-6fff-4f7c-8d2a-574ceea26347. Wait time elapsed: 18.302 Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id 6ac9414d-6fff-4f7c-8d2a-574ceea26347. Wait time elapsed: 18.303 Jul 01 09:47:44.965221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:47.628598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.668s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:47.629215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:47.631195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:47.631604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:47.631946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:47.633182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6f024d6-475e-485e-ad32-506fe0ce7b81]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.644502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb6d03a6-a820-48eb-96d9-f0b806d9756b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.668698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[553cbc83-7aa1-4b23-b5f9-fca56bea2e5a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.701719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ab448ef-1f76-4287-83c3-9b187c7d064a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154854, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.733520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83d66fe7-91f4-4c75-82d4-ed9f90dff01c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.753051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e76f1d1b-8edc-4c03-acf6-f4edfc46c1a1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.773608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77172672-2976-4700-837f-774c115abae3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.794805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21fdc3c9-af59-4fa0-95df-2d33d7dbb4cf]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.795621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:47.795961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:47.796310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:47.796647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:47.811960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:47.812789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80fdbc80-37e9-4071-86ae-89b65ed15bcd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.816019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2d4a99a-6fb8-4ed4-b9f7-6715bcdafddc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.817431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:47.845548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:47.846237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:47.846631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:47.847017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:47.847434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:47.883180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c15369b4-7aba-40b6-97ab-b35525f435a1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.928639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4ffb7e0b-0187-4375-9a67-650d647fe2c6]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:47.928639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:47.928639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:47.928639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:47.928639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:47.928639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:47.928639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:47.928639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:47.928639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:47.928639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:47.928639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:47.928639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:47.928639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:47.929766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ffb7e0b-0187-4375-9a67-650d647fe2c6]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:47.930495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:47.932532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:47.935011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e32d42b1-97c6-4308-8530-fcc66347fd03]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.935705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:47.936107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:47.936361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:47.936757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:47.937843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14abba7b-4e79-4106-a62c-24d99f91785f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.938769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6da3236a-42a8-49ac-b9ec-a9c12cf04ad7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.939691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d74ed5c3-a496-4d0c-b85a-72df6b27587b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.958580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8149760c-7b58-4861-9abc-5b43f1ca2499]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 376396, 'reachable_time': 27228, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154872, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.963967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:47.963967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88f6e141-1633-4f55-8861-1856092d13f0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:47.965085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:46Z", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:47.970223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a33bd4dc-b242-4ec4-82f5-aaf86f711a3c. Wait time elapsed: 18.672 Jul 01 09:47:47.989975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a33bd4dc-b242-4ec4-82f5-aaf86f711a3c. Wait time elapsed: 18.672 Jul 01 09:47:47.989975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:48.770239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.803s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:48.771618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:48.772691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:48.773119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:48.773499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:48.775228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2388c868-1789-4c4c-8f51-41714c35c047]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:48.794214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bd0e5ec-4b07-4447-ad67-751aef595582]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:48.817397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[044023de-b071-4088-945a-b4a2dac99e11]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:48.857782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74417ff0-2563-4161-8a7b-ccd065a12a3b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154875, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:48.887927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c149490d-ead1-4da8-a09b-925ec18021c5]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:48.912544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d206c8f8-fa7f-4481-b865-fa8b3ee13049]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:48.933414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b98638fb-8449-4893-a001-7b96d80f6d64]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:48.955884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35042cdd-feb6-48a8-92e1-5069b7049c5c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:48.956747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:48.957118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:48.957498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:48.957865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:48.975182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:48.975629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dda94eea-b8d3-4560-a0c7-e03dc6580282]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:48.978484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0260118d-fb87-44d4-b03e-45cd6f8b712c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:48.979587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:49.018227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:49.019489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:49.021139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:49.021873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:49.022922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:49.078866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0b4098c-24fe-48e7-90b6-fd9d04d77c4d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:49.114638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[222a3e72-63de-45f6-8261-479f359fb4de]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:49.114638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:49.114638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:49.114638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:49.114638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:49.114638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:49.114638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:49.114638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:49.114638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:49.114638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:49.114638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:49.114638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:49.114638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:49.115766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[222a3e72-63de-45f6-8261-479f359fb4de]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:49.116078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:49.117952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:49.120402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2688c351-01dc-41f3-a01d-cb990cbb2c82]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:49.120402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:49.120402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:49.120402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:49.120763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:49.121859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5632be1-05fc-47e0-bb7e-de7f0b73ec05]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:49.122883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d33d8470-8f3a-41cc-9ba4-e6280213d86e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:49.123996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb197542-a97e-440b-b63f-95e54e9aac16]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:49.153429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd4797e4-eeda-445e-94ce-b1dc4dc6ffa9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 376512, 'reachable_time': 17944, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154894, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:49.163405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:49.164648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af8f3d69-5019-47d8-96a1-6c5ebe9823d8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:49.165879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:48Z", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:49.169974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id 6ac9414d-6fff-4f7c-8d2a-574ceea26347. Wait time elapsed: 22.510 Jul 01 09:47:49.185613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id 6ac9414d-6fff-4f7c-8d2a-574ceea26347. Wait time elapsed: 22.510 Jul 01 09:47:49.185613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:50.948164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-46dab182-3256-4e4b-b4d8-8e962cc0eb16 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['19b5d892-dd48-419b-8faf-b3c1e06f0f75'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:47:51.964385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.797s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:51.965046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:51.966163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:51.966682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:51.966972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:51.968239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8f68447-3d83-4202-aa8f-d9b79401157e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:51.980793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a926c5c-d138-4f32-b8c2-9989ac04bc49]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.001443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51b1c04e-849a-40d8-a978-94dbe6bb3e8e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.029570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ba2a9fa-e812-4c1c-b21a-81a4419857c6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154944, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.051582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59d38718-7e96-4aaa-8a96-0aac00eb3536]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.074447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5c01248-79b2-4bae-8cc3-c58053572413]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.096998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[935e798b-848b-46b7-a747-ba93e95feac0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.117865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3780d9f9-41f3-429a-86f9-19ae7cc389f8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.118490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:52.118772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:52.119039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:52.119319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:52.133973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:52.134483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddf01eac-1f75-4f97-93d0-dd66506ed5e7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.139872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd1c58d4-502f-4459-8403-1796a0554b88]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.141093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:52.169412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:52.170255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:52.170659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:52.171051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:52.171494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:52.207002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c8b3833-9c78-414e-b2b2-652be8efd26d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.245563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8d1c4092-0fe9-49f8-ba46-a5b5a643f1e0]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:52.245563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:52.245563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:52.245563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:52.245563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:52.245563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:52.245563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:52.245563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:52.245563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:52.245563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:52.245563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:52.245563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:52.245563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:52.246584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d1c4092-0fe9-49f8-ba46-a5b5a643f1e0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:52.246846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:52.248900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:52.251237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[993f1808-57d6-4d19-9725-b4d57fc5b0ea]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.251390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:52.252104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:52.252104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:52.252417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:52.256250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d36f3bc1-b333-4bcf-a116-ca8390fcd0fd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.256250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b99da121-a9c2-4510-9888-9d1801c30770]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.257519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df902694-85f0-4f75-b3ab-bce2ad0f129a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.279698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63c955d8-9c7b-4429-b39a-585ac1725b42]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 376829, 'reachable_time': 34468, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154963, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.284399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:52.284399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce836d9e-2c33-4aa5-9c64-7f884e2ae025]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:52.285488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51ba8143-b208-427e-a205-dbba72b03f64", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.39", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "243839bf-9a38-4d45-b708-e65090ba4f3e", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2707, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-1329265588", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:8c:b4", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e19a7f84-1351-4426-bdc9-cb94d455b0cd", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:49Z", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:49Z", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:52.289236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:49Z", Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3 Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 1, update_id a57c3b8a-da84-49e7-96f4-84fcdaec01b2. Wait time elapsed: 1.341 Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 1, update_id a57c3b8a-da84-49e7-96f4-84fcdaec01b2. Wait time elapsed: 1.342 Jul 01 09:47:52.302701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:53.075159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.785s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:53.076111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:53.077490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:53.078275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:53.078401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:53.080244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13977fad-dc3a-4700-a132-05fe88459a7f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.096737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2dc3c67f-725c-49cc-a2c1-ff3dcf1dc8c3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.120348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8330a01-4dc7-4e46-8bbe-de5f07a7b41b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.152952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fe2755b-4cd0-4bf4-9e20-d4ee8961a1f0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 154966, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.175349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7eac5e08-81d1-4fb7-b060-703ee7070e49]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.186672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa45dd32-2c38-4fec-b93c-949b3e268775]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.210371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3f92b75-8469-4aaa-8f0f-62596b9bb6fe]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.231845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b158310-1434-43cd-a931-8d0e914749b4]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.232612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:53.232934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:53.233202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:53.233486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:53.249369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:53.249646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c8ba6d3-5253-4614-8d5e-77e54c29ed0e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.252433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3254d682-29a4-452e-b64d-ed73c4a3047b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.253386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:53.289969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:53.290993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:53.291851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:53.292718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:53.293355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:53.337688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7b197c9-4e69-4956-8db5-6398cee3ec5b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.372769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8ee828af-beb4-4f6c-9f2e-f474b534825b]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:53.372769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:53.372769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:53.372769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:53.372769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:53.372769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:53.372769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:53.372769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:53.372769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:53.372769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:53.372769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:53.372769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:53.372769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:53.373825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ee828af-beb4-4f6c-9f2e-f474b534825b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:53.374728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:53.376789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:53.379321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1df8155b-20dc-493e-8731-9030408dfa8b]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.379321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:53.379321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:53.379321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:53.379321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:53.380680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb104cf6-c60d-4c52-a872-369a458a09b0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.381761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b93f0ada-999b-423d-9daf-0ce01da295e9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.382813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f31f4530-c146-4cc8-be3c-b91ee4743804]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.403504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80544467-7eca-4e14-b4ad-222e325af0a0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 376941, 'reachable_time': 21064, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 154985, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.416746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:53.417561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9719565d-aac7-453f-b3be-32f4118c436a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:53.418843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51ba8143-b208-427e-a205-dbba72b03f64", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.39", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "243839bf-9a38-4d45-b708-e65090ba4f3e", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2707, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-1329265588", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:8c:b4", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e19a7f84-1351-4426-bdc9-cb94d455b0cd", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:49Z", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:49Z", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:53.423028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:52Z", Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a33bd4dc-b242-4ec4-82f5-aaf86f711a3c. Wait time elapsed: 24.129 Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a33bd4dc-b242-4ec4-82f5-aaf86f711a3c. Wait time elapsed: 24.130 Jul 01 09:47:53.436751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:56.376634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.952s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:56.377812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:56.379187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:56.380663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:56.380955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:56.383018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5ea41c0-5de6-4901-8e6e-a1372eb4d52d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.395853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2817fa6b-618b-441e-b2ea-d769c85addfa]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.421615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf8904af-e887-4cfc-a363-a0e6b9bcc6ea]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.476117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[598dd44c-bd06-41ed-9b9c-1a0a3e74c9df]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155036, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.497859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49b438da-4da5-43cf-885e-c487572eb50b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.521252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bec1d25f-9a0d-47c9-bee5-bec8ab78e154]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.543960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d75f7dc4-ef97-4ddb-b0ec-df2551b5588b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.564802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6bb141b-f35a-4de1-a4c2-4f863b94d8ca]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.565410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:56.565734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:56.566004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:56.566278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:56.584841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:56.585280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b507661e-6f44-421c-883b-9d7aa0269c3f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.588106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea4ff7fb-ce82-4e45-9013-5ee0f54ac917]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.589224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:56.622026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:56.622993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:56.623583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:56.624123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:56.624855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:56.661094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ece7bd6-7c15-4c05-aead-4eab1c520ce9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.698164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[96f893d0-3231-4bbe-9494-d3cfd610c4ac]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:56.698164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:56.698164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:56.698164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:56.698164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:56.698164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:56.698164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:56.698164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:56.698164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:56.698164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:56.698164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:56.698164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:56.698164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:56.699664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96f893d0-3231-4bbe-9494-d3cfd610c4ac]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:56.700850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:56.706985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:56.711988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b814401c-17bb-4484-81bc-1d6dd9f1d144]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.713064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:56.713858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:56.714257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:56.714736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:56.715943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03703876-39ac-4550-94bd-a9d59786d1e0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.717016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2398bfeb-1243-48b2-946c-b9662bdd2092]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.718083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d244995-a5ad-4506-8315-3f9ebb97e54f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.740533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e9e4344-a9cd-4f3c-8544-386ac74747f4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 377274, 'reachable_time': 33080, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155055, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.746565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:56.746565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79052776-012d-4504-9990-82a11b29d28e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:56.747294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51ba8143-b208-427e-a205-dbba72b03f64", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.39", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "243839bf-9a38-4d45-b708-e65090ba4f3e", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2707, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-1329265588", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:8c:b4", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "" Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:49Z", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:49Z", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:56.750054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:52Z", Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3 Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a57c3b8a-da84-49e7-96f4-84fcdaec01b2. Wait time elapsed: 5.800 Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a57c3b8a-da84-49e7-96f4-84fcdaec01b2. Wait time elapsed: 5.800 Jul 01 09:47:56.777289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:47:57.654498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.905s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:47:57.655102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:47:57.655914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:47:57.656303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:57.656646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:57.657876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d0bddc5-98c3-402f-b763-f203885f64f5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.669630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d927b9f7-6267-45ae-8284-04c92b7059a4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.691379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bf4116d-29fe-4524-80d8-20db826a70d3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.721043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6411ddb-1acd-4c4c-b3c9-932aebda4a01]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155059, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.741626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4371dfdc-d9ae-4052-b930-69c74f82f837]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.762123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a15b69f9-e6f3-4f9c-ab56-ff88e025af56]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.782574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1662ce9c-2182-4e42-82ad-48f42c9c5073]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.803143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e3dac9d-258e-4398-8875-32c6a62c00c1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.803709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:47:57.803981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:47:57.804243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:47:57.804515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:47:57.819124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:47:57.819480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66b08b59-f1ad-42f0-b688-40fefd74b1df]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.822136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c25b1e63-b4aa-4b25-9bae-311b1f1a83fe]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.823229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:57.870185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:57.870854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:57.871558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:57.872099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:57.872645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:47:57.927772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4aec29ec-f8b7-404c-bbe8-3e8f714b6401]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.959848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1d00f121-6e39-4417-a08e-1180fa6f4e4d]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:47:57.959848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:47:57.959848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:47:57.959848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:47:57.959848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:47:57.959848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:47:57.959848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:47:57.959848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:47:57.959848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:47:57.959848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:47:57.959848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:47:57.959848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:47:57.959848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:57.960808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d00f121-6e39-4417-a08e-1180fa6f4e4d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:57.961058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:57.964545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:57.966624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0289dbbe-46a9-41d1-8e1d-d87e4d64015b]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.966975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:57.967536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:57.967797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:47:57.968140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:47:57.969235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29f52fa9-6360-4016-9e5a-c7d0a7004376]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.970297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8376cafb-2aac-4cc1-94dd-1dcdcb4e9e1a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.971319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06754adc-77fa-4d6a-b49a-b9ecf6f86357]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.991785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba51be6d-d813-4d53-a1e9-c1032303d1b1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 377398, 'reachable_time': 18484, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155079, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7cb95b6e-9211-45e8-8830-3423e1c54f68]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:57.996612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51ba8143-b208-427e-a205-dbba72b03f64", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.39", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "243839bf-9a38-4d45-b708-e65090ba4f3e", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2707, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-1329265588", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:8c:b4", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "" Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:49Z", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:49Z", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:56Z", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 22, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:47:58.001691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:58.013728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:47:58.013728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:47:58.013728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:47:58.013728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:47:58.013728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:47:58.013728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:47:58.013728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:47:58.013728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:47:58.013728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:47:58.013728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:47:58.013728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a57c3b8a-da84-49e7-96f4-84fcdaec01b2. Wait time elapsed: 7.049 Jul 01 09:47:58.013728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a57c3b8a-da84-49e7-96f4-84fcdaec01b2. Wait time elapsed: 7.051 Jul 01 09:47:58.013728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:01.009206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.007s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:01.009831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:01.013426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:01.013796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:01.014083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:01.015562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e99ea50a-af19-4795-9eaf-5af8bdf8cd92]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.028608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4d38b3e-1923-4dfc-9ee7-39545f23d25c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.048281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ac05727-065f-4d6f-83b2-0acf4f6bc078]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.074282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d2cdfaa-1693-4d4b-9241-9c7f2c1da148]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155134, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.095477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4730399f-2480-4bb3-8554-9e40dc5982c7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.115651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a265b137-9af0-4419-8d76-0454566dc65c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.135453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eaedcd0c-3541-4e89-838a-16028fe2dbff]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.155492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4a35c53-4c9a-49cc-99dc-04e15bc87257]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.156160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:01.156501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:01.156834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:01.157122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:01.171830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:01.172096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac8e9b7f-2e03-4816-ac6b-97f761d57647]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.175159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b58da384-f9e5-4e87-b524-8abfb2d9ef79]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.176196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:01.221015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:01.221691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:01.222254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:01.222719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:01.223194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:01.269140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa4e749b-be3e-44e5-b712-5609c00d4fbf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.302257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[78e94447-a749-4ec6-9682-238494b208fe]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:01.302257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:01.302257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:01.302257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:01.302257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:01.302257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:01.302257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:01.302257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:01.302257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:01.302257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:01.302257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:01.302257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:01.302257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:01.303868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78e94447-a749-4ec6-9682-238494b208fe]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:01.304863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:01.306854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:01.309266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[569b1c00-7757-4929-b05f-0192caaee9f7]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.309266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:01.309266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:01.309553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:01.309745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:01.310720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77e0511c-b5da-4960-a030-4b54df17474e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.311657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ad24851-7914-44a5-84c8-d9f93223e97e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.312965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e89466c7-c2f6-44b2-8907-bd7d2eb31784]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.333557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e327455c-3694-4ae7-a49d-98d995bc8bce]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 377733, 'reachable_time': 30072, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155152, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.341308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:01.341671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b6b3f46-ef94-45e8-a38e-04c84326c453]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:01.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51ba8143-b208-427e-a205-dbba72b03f64", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.39", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "243839bf-9a38-4d45-b708-e65090ba4f3e", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2707, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-1329265588", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:8c:b4", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "" Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:49Z", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:49Z", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:01.345983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:58Z", Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 24, Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a57c3b8a-da84-49e7-96f4-84fcdaec01b2. Wait time elapsed: 10.396 Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a57c3b8a-da84-49e7-96f4-84fcdaec01b2. Wait time elapsed: 10.397 Jul 01 09:48:01.362832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:02.124932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.779s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:02.125533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:02.126144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:02.126465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:02.126786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:02.127943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83fafffa-88e9-45db-9ec5-bdf3b327bb09]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.140351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52747919-b742-40f0-9a1d-7ab667c2e554]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.161712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35805f44-ebe6-4986-8ab1-e52023c4cbb9]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.190193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50987679-e5fc-44c9-a447-015fb9150885]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155155, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.211821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eac7ff46-2f4f-455c-8e92-512979593a2f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.233986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c07dd424-a93d-4374-9dc9-f1acce3aaad4]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.258984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4ef17c2-74a2-4d37-a77a-15453c1476ed]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.281677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa537164-eecc-46a6-b9f5-dc248468ce49]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.282754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:02.283238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:02.283829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:02.284394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:02.310346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:02.311712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ac5c498-c0dd-43e5-9c1c-4fe0b1744b22]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.317001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed8e4785-f7ad-43d9-98a3-a43e9c8fda76]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.319382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:02.358678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:02.360606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:02.361065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:02.361747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:02.362226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:02.423577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd2f0623-86b0-4944-a186-adc14a03a657]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.470602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[13952138-d5c5-4702-9ad4-37941c7ce651]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:02.470602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:02.470602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:02.470602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:02.470602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:02.470602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:02.470602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:02.470602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:02.470602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:02.470602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:02.470602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:02.470602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:02.470602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13952138-d5c5-4702-9ad4-37941c7ce651]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:02.472003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:02.474423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:02.477761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39bf3396-d93e-40e5-acc2-721ea9c47833]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.477761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:02.477761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:02.478789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:02.478789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:02.482987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4bfc095-473c-4255-bfa4-5aaa7e330cd1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.485640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9af3f4df-9c4a-4f3f-b9a4-aed902f0fc80]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.490238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[669eb612-1bc0-4694-9480-e60f3aeba304]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.519865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b561569-e825-4780-81e2-47aa24d82465]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 377845, 'reachable_time': 41652, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155174, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.531448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:02.532106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d11e033-87c0-4dc7-8100-851048401ba4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:02.533429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51ba8143-b208-427e-a205-dbba72b03f64", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.39", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "243839bf-9a38-4d45-b708-e65090ba4f3e", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2707, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-1329265588", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:8c:b4", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "" Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:49Z", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:49Z", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:02.537699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:00Z", Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 25, Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a57c3b8a-da84-49e7-96f4-84fcdaec01b2. Wait time elapsed: 11.591 Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a57c3b8a-da84-49e7-96f4-84fcdaec01b2. Wait time elapsed: 11.592 Jul 01 09:48:02.550008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:05.314438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.774s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:05.315099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:05.315724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:05.316151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:05.316432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:05.317594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[363bd4d5-2b09-4e22-b8d4-e674f0c0810d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.331324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ef312b6-db51-48ff-8cbc-389954fd47c0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.351974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[daec1c49-e327-4e52-aaea-c737809227d1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.377647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb48dc87-6680-4415-82f9-5dde7b88c6f9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155224, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.399212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0dca4e8b-0ffa-4c8d-8e62-1244b00ca0d6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.419835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a1c946a-f6ee-4c98-b259-6565844614bb]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.440640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e91a881-aa87-4fad-92ec-c90fe686c57a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.461211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcfa0267-3519-47dd-a32e-fa3d13eabadf]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.461754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:05.462037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:05.462305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:05.462587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:05.478215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:05.478495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc6d0c7c-bb83-4b57-a218-6f92a1cf9ebe]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.481156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9b72884-a672-4891-9031-6bb9d05756fa]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.482120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:05.524236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:05.524913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:05.525308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:05.525696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:05.526084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:05.563827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd567fa9-5d26-45f9-8184-199d8cf4ae10]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.604070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e0bc0e49-93e3-408e-8bb0-83bac1b5acca]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:05.604070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:05.604070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:05.604070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:05.604070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:05.604070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:05.604070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:05.604070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:05.604070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:05.604070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:05.604070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:05.604070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:05.604070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:05.605139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0bc0e49-93e3-408e-8bb0-83bac1b5acca]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:05.605785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:05.607637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:05.609649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c5c2729-49ee-4753-9610-4337da31e9ce]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.609649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:05.609649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:05.609649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:05.609649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:05.610900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a96baf28-de04-452a-a10c-ce4b8c85975b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.611928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50a8de8e-d723-46c2-aab0-7a33c05d5ac0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.612892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a7b486f-54a8-47fd-bbe3-f5e11ac52331]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.637205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ededcb0b-b12c-4526-8206-607f600ebeb8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 378164, 'reachable_time': 38256, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155247, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.643098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:05.643525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dcf52839-407c-4264-972d-94a49ad37b69]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:05.644627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51ba8143-b208-427e-a205-dbba72b03f64", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.39", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "243839bf-9a38-4d45-b708-e65090ba4f3e", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2707, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-1329265588", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:8c:b4", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "182c8477-e01b-4a79-88be-990d13451f0d", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:49Z", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:49Z", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:05.647601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:03Z", Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 26, Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a57c3b8a-da84-49e7-96f4-84fcdaec01b2. Wait time elapsed: 14.698 Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id a57c3b8a-da84-49e7-96f4-84fcdaec01b2. Wait time elapsed: 14.699 Jul 01 09:48:05.658989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:06.589762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.942s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:06.590332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:06.590943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:06.591273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:06.591568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:06.592980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[808cd056-5473-4477-a13c-7ea9218acad4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.611158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d82a9fe-02ae-4abe-a335-f5330ad22dfc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.633896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5aad0db-cd18-4b43-8b85-68bbe55de9d4]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.668304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d332d611-d6b6-4836-9051-4520ce85a64d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155250, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.691776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f2d9b0e-7cbe-4cb9-97a1-382fc6c30f70]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.718516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd53ac18-a67a-4365-91e8-c7a3d70c3101]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.742890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d494caf-614a-4674-884e-86407e5152ec]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.766864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22623419-ee6f-4c51-b82c-ad45ce004296]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.767986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:06.768264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:06.768554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:06.768835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:06.790141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:06.791462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a116cc2-e2bf-4509-b04e-20d31e6577a8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.794284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0174480e-086c-4ee2-829e-ab376255dca3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.795222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:06.837302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:06.840503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:06.841371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:06.842184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:06.842986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:06.912912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc809371-7a22-46ad-aef3-7aded32b131d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.956896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8136c6d1-d8cc-4c71-9f45-0c9d4fb7760f]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:06.956896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:06.956896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:06.956896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:06.956896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:06.956896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:06.956896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:06.956896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:06.956896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:06.956896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:06.956896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:06.956896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:06.956896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:06.958522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8136c6d1-d8cc-4c71-9f45-0c9d4fb7760f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:06.959018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:06.960951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:06.963281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22d2abce-0b6f-4dd1-8168-774c69595d1d]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.963281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:06.963281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:06.963281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:06.963962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:06.964974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a35c334-4503-4c21-ac6b-ff5c46c80694]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.965934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5f4f306-fab3-49d3-b8a9-7d0b390095c1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.966879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50c80162-326d-4d60-8a43-7acad544d070]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.986622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[caf82c0c-192e-48ba-bdd5-9383212e759a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 378293, 'reachable_time': 15804, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155269, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.991142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:06.992403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b94f605-e604-4236-86c8-2d5261ef3605]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:06.993579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "51ba8143-b208-427e-a205-dbba72b03f64", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.39", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "243839bf-9a38-4d45-b708-e65090ba4f3e", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2707, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-1329265588", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:53:8c:b4", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "182c8477-e01b-4a79-88be-990d13451f0d", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:49Z", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:49Z", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:06.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:05Z", Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 27, Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:07.013211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3 Jul 01 09:48:15.096137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:15.096862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:15.096862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:15.097056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:15.152275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:48:19.818200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-75e06dd1-bdda-4d17-a6c1-4fd82ed92e4b tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['19b5d892-dd48-419b-8faf-b3c1e06f0f75'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:48:19.819444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 1, update_id 53c2b2f3-efc6-42c4-a4ce-a66edea9a45c. Wait time elapsed: 0.001 Jul 01 09:48:19.819784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 1, update_id 53c2b2f3-efc6-42c4-a4ce-a66edea9a45c. Wait time elapsed: 0.001 Jul 01 09:48:19.820212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:21.450976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.631s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:21.451597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:21.452486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:21.453088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:21.453372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:21.454568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b69a55f-f7bb-487e-9451-eec1dab0483a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.476202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c874ff46-abfc-48a9-8505-806d6a079767]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.497717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cd5b917-b4b8-492a-82e8-32ad5afa1ef5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.537667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b251b2fa-2946-4db1-a25d-985ebb8768fa]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155497, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.573296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[486993d2-b822-4832-857d-f3a89bf55ac3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.611773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d18cce94-fcaf-456b-afc6-2d2177a4d9b1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.636546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee861843-d215-4a5b-b94c-59a8b2633d49]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.660980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a9bfd9d-3243-40db-a2d8-09cc04255483]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.661903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:21.662182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:21.662444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:21.663037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(a387f7c3-8eb7-414d-b5d6-3f996b9786f3) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:21.679666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:21.679923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87632e28-f33b-44dc-895a-baea6b09bf6c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.683466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19d64e6a-2712-49cd-812e-7c6fbd041e49]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.685140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-a387f7c3-8e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:21.712867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-a387f7c3-8e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:21.713857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:21.714445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:21.715169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-a387f7c3-8e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:21.715587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-a387f7c3-8e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'a387f7c3-8eb7-414d-b5d6-3f996b9786f3', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:12:be'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:21.748103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ea50551-c316-4f8e-8c21-191b2449f75b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.795162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d2737c56-8ca7-4790-b4a4-c5595379ea0a]: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:21.795162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:21.795162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:21.795162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:21.795162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:21.795162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:21.795162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:21.795162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:21.795162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:21.795162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:21.795162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:21.795162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:21.795162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:21.796817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2737c56-8ca7-4790-b4a4-c5595379ea0a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:21.797787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:21.800396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:21.803903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12de5d68-1a6e-4196-980d-f5622c3fc6d9]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.803903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:21.804547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:21.804827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:21.805370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:21.807230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd154f3d-3a51-4f26-8b9c-de5af5322446]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.808420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee47270b-900e-4328-ab5a-dbea39f4f394]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.809327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6175e3b3-a385-412e-b72c-4f290b7e54b9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.838367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d1f4839-ff7e-45e3-acf7-058fa5cbc439]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 379779, 'reachable_time': 22068, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155517, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.845746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:21.846000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3885ad4-8838-4828-9a93-863c31bf1993]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-a387f7c3-8e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:21.847252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a387f7c3-8eb7-414d-b5d6-3f996b9786f3", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "11874ba8-fb22-45c0-a33a-5129d93d252b", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:12:be", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2680, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:28Z", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:08Z", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a12d3149-e34c-4cbd-8d1b-52ac56906007", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:21.850904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id 53c2b2f3-efc6-42c4-a4ce-a66edea9a45c. Wait time elapsed: 2.031 Jul 01 09:48:21.872498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 2, update_id 53c2b2f3-efc6-42c4-a4ce-a66edea9a45c. Wait time elapsed: 2.032 Jul 01 09:48:21.872498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:22.089510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b242be44-fba7-498e-be6c-3f3ec4cd2b74 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['19b5d892-dd48-419b-8faf-b3c1e06f0f75'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:48:22.529197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.678s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:22.530157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:22.531970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (19b5d892-dd48-419b-8faf-b3c1e06f0f75), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:22.532304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:22.532770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:22.533998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fd730fb-ece9-4a14-ac37-2f53e13dbdc5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:22.554819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40e94efe-bd86-4c49-bf04-7abe0fb54034]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:22.580301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f348887-caeb-454c-a33c-a1067e97392f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:22.621687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[528a62da-624e-447b-933b-61f2ad63ef31]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155520, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:22.649438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e8d9c59-314a-4d6e-9027-0b6d1fc50ac8]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:22.675751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[468f5853-5570-47f9-be7d-2599c74104ec]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:22.698327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2057dcd-f639-4120-a8ac-7bc19126f43c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:22.731067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f4a47df-07d9-420e-bc1f-0e7a5dbba388]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:22.731946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:22.732367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:22.732779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:22.752766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fd45c94-9657-4347-ba9a-16db5f688585]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 379888, 'reachable_time': 27560, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155530, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:22.753782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:22.754605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:22.755689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:48:22.755960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:48:22.756300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:22.756569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:22.756869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '57e44b67-0e95-4913-9568-23b3f88dd1b4', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:48:f4:e8', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'device_owner': 'network:router_gateway', 'standard_attr_id': 2678, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.248', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:47:25Z', 'updated_at': '2022-07-01T09:47:28Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-57e44b67-0e), ns(qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:48:22.776979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-57e44b67-0e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:22.777266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97ddbfb2-9699-4588-957f-46b19fc810f5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:22.779983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e52566b7-bebf-460b-ac93-77ebbb7c8604]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:22.784350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-57e44b67-0e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:22.784992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:48:22.788273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-57e44b67-0e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:22.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-57e44b67-0e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:22.789864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-57e44b67-0e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:22.790372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-57e44b67-0e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:22.790845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-57e44b67-0e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '57e44b67-0e95-4913-9568-23b3f88dd1b4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:48:f4:e8'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:22.850910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3bbf6d5-a4a2-42d3-a14a-7134cc7cf060]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:22.896063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7ee86874-c05a-4f8b-846d-f95dcff2e5c8]: Network interface qg-57e44b67-0e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:22.896063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:22.896063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:22.896063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:22.896063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:22.896063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:22.896063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:22.896063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:22.896063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:22.896063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:22.896063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:22.896063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:22.896063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-57e44b67-0e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:22.897137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ee86874-c05a-4f8b-846d-f95dcff2e5c8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-57e44b67-0e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:22.897137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:48:22.897429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:48:22.929161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aefa8c00-2dbe-4894-a80a-eaac40dfb595]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:23.114855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0707a1da-457f-4848-a1cc-0c9dfad9e93f]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:23.127048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b998872-598c-436a-bc10-85974b3946e1]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:23.303036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39f9256e-5dd7-48bb-9657-5ae4c677aad9]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:23.303720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:48:23.304146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:48:23.304832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:48:23.305194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75" released by "process_external" :: held 0.549s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-57e44b67-0e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-57e44b67-0e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-57e44b67-0e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:23.306162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-57e44b67-0e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-57e44b67-0e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:23.308772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:23.311768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04a4ef3f-d7a0-4e51-9b70-d60f039c70e3]: (4, ['qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:23.311768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:23.312201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:23.312466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:23.312884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/19b5d892-dd48-419b-8faf-b3c1e06f0f75.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:23.314366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f93c01cd-23b2-4e14-b053-7de7824bff41]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:23.315490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6cbf77ad-8702-4218-a4b5-a5bd289b9135]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:23.316443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b88456fe-728f-48c1-b55d-1233b04f1fac]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:23.339221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf3e71f7-9f2f-4138-a4e0-e776fe3bed77]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 379888, 'reachable_time': 27560, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155574, 'error': None, 'target': 'qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:23.345394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:23.347255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbb59c50-026f-4e83-953b-c4d811c182d0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 19b5d892-dd48-419b-8faf-b3c1e06f0f75: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-57e44b67-0e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-57e44b67-0e not found in namespace qrouter-19b5d892-dd48-419b-8faf-b3c1e06f0f75. Jul 01 09:48:23.347709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1101530042", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248" Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:21Z", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57e44b67-0e95-4913-9568-23b3f88dd1b4", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:f4:e8", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19b5d892-dd48-419b-8faf-b3c1e06f0f75", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2678, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.248", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:47:25Z", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:47:28Z", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:48:23.351908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:23.361757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 1, update_id 348b5a45-c9fa-4824-9e81-95ba22b44e99. Wait time elapsed: 1.261 Jul 01 09:48:23.361757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 3, priority 1, update_id 348b5a45-c9fa-4824-9e81-95ba22b44e99. Wait time elapsed: 1.262 Jul 01 09:48:23.361757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:23.668712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-5a94ae6e-c20f-444a-a961-a4a688cce8f0 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got router deleted notification for 19b5d892-dd48-419b-8faf-b3c1e06f0f75 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:48:23.761434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.408s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:23.761910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 was not found. Performing router cleanup Jul 01 09:48:23.763072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2dd07ab-ed60-4148-b649-5020d53db8a5]: (4, ['qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:23.763916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:23.764287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:23.764628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, update_id 348b5a45-c9fa-4824-9e81-95ba22b44e99. Time elapsed: 0.413 Jul 01 09:48:23.765020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 1, priority 1, update_id 0539361a-aa6e-4118-8dbe-8bf30674fcd0. Wait time elapsed: 0.096 Jul 01 09:48:23.765348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, action 1, priority 1, update_id 0539361a-aa6e-4118-8dbe-8bf30674fcd0. Wait time elapsed: 0.096 Jul 01 09:48:23.765688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 19b5d892-dd48-419b-8faf-b3c1e06f0f75 was not found. Performing router cleanup Jul 01 09:48:23.766748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c1c9c6a-9d1f-4c01-bb50-6ae7c15c05a1]: (4, ['qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:23.767883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:23.768247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:23.768581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 19b5d892-dd48-419b-8faf-b3c1e06f0f75, update_id 0539361a-aa6e-4118-8dbe-8bf30674fcd0. Time elapsed: 0.004 Jul 01 09:48:26.052101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-0753a156-9f1a-4754-a29d-8d44e5488f85 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['a6bfbb45-cc90-4764-a46d-cff9e14d2acc'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:48:26.056653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 1, update_id 169688c0-a1e6-41ac-a46d-80f52e392a1a. Wait time elapsed: 0.003 Jul 01 09:48:26.057100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 1, update_id 169688c0-a1e6-41ac-a46d-80f52e392a1a. Wait time elapsed: 0.004 Jul 01 09:48:26.058724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:27.325140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.267s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:27.325740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:27.326456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:27.327045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:27.327506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:27.328669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[579eff9c-43b0-4661-9187-b83be7fb90e8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.350901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15089b95-fffe-4383-8a86-99834cd89238]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.371712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77b84e8f-d77e-4872-b2d3-33c97d70f650]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.409744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbada058-2e72-4e4c-a795-900cd42d9b24]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155611, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.434209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19913802-c741-46bb-bec5-585e7715a6e0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.458241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d603b57-aa43-42aa-9a36-9542d2ea9cf3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.482455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a7e00e4-d49e-40a9-8885-ca1658520b17]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.505105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[061f545a-d231-4793-b6e4-b58ad474d72f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.505951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:27.506227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:27.506514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:27.538264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3296cca2-6b56-42af-8647-2a9ed5e34e8b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 380367, 'reachable_time': 18528, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155617, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.539614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:27.539993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:27.540562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:48:27.540906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:48:27.541429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:27.541747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:27.542098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'f9aeffe6-476c-468a-844a-926e714fc4c1', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:b3:9f:b7', 'admin_state_up': True, 'status': 'DOWN', 'device_id': 'a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'device_owner': 'network:router_gateway', 'standard_attr_id': 2746, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.164', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:48:24Z', 'updated_at': '2022-07-01T09:48:26Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-f9aeffe6-47), ns(qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:48:27.560810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:27.561160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f072b071-87d3-4775-bef5-20f375ea4bf5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.564327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23feda87-7a20-449e-960d-ad57e15b7423]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.565350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-f9aeffe6-47, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:27.565736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:48:27.566616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-f9aeffe6-47, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:27.567406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-f9aeffe6-47, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:27.567871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-f9aeffe6-47, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:27.568324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-f9aeffe6-47, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:27.568931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-f9aeffe6-47, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f9aeffe6-476c-468a-844a-926e714fc4c1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b3:9f:b7'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:27.614030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0caf10b0-16d1-4512-ae8a-712dc5258cf6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.659408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[95e83679-e796-4a5c-b32c-5ad1f6ff860a]: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:27.659408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:27.659408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:27.659408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:27.659408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:27.659408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:27.659408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:27.659408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:27.659408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:27.659408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:27.659408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:27.659408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:27.659408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:27.661476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95e83679-e796-4a5c-b32c-5ad1f6ff860a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.662154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:48:27.662951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:48:27.676064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c4b07bc-7e90-474b-b7c0-4acd9b4841a6]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.870211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27b3ab0a-a47f-4b3a-a8f3-2f84d0559af9]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:27.906129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5ae1721-7baa-49bb-b23a-c33cf5a990dc]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:28.122203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4cb9114-c277-43e3-bb25-df517c06a8b4]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:28.122749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:48:28.123062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:48:28.123375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:48:28.123675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" released by "process_external" :: held 0.583s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:28.124457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:28.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:28.130948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80d14d72-e063-41e7-a6c9-83537b8baa9f]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:28.130948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:28.131894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:28.131894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:28.132075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:28.133226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae9e7de3-6d56-43b8-a4a0-5e43478266b6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:28.134470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e3aba71-b429-4b33-a2d4-7ae5df4f5894]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:28.135475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33946edd-29f5-446d-9457-eb268265580e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:28.162887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d76ae6c2-4fd4-40cd-9303-9aebcaec44c4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 380367, 'reachable_time': 18528, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155664, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:28.169556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:28.169840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cc8ef72-62c1-4a1b-a74d-8b840b0371f6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:28.170942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:25Z", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:26Z", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 169688c0-a1e6-41ac-a46d-80f52e392a1a. Wait time elapsed: 2.119 Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 169688c0-a1e6-41ac-a46d-80f52e392a1a. Wait time elapsed: 2.119 Jul 01 09:48:28.176751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:28.754013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-aac4faf3-c182-4957-afca-ee108ca1d88e tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['a6bfbb45-cc90-4764-a46d-cff9e14d2acc'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:48:32.169035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.996s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:32.170221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:32.171603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:32.172208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:32.173008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:32.174457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1ab3a43-b475-452d-a7f5-71e589d68a52]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.214335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c136b1c6-20fa-4a08-b40a-fb8fa0bd4623]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.260668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[037cf550-616b-4326-bcd5-3a826b50c94e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.359078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64808fb8-4929-4278-8c77-f1927a25e71e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155676, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.444123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad5a2224-aceb-44a8-84e1-8bbff38b7f95]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.503394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0738ee67-1b9d-4bb2-a932-800245ee8a6f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.540927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d78db92-a56d-4e2c-947c-808585ad8bac]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.575796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9950abb4-8e98-4768-892d-e1fb333416f5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.576409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:32.576683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:32.577218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:32.577509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:32.614653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:32.615139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a88d2d93-fb09-4c70-aa7f-2ad5f8812c01]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.619106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3402ec4-d6ef-42ba-a381-9f759b9aea14]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.620196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:32.620686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:48:32.622533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:32.623383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:32.623963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:32.624519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:32.625077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:32.670974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3e4f42b-6e87-4d16-9eaa-6f19b42c4198]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.732136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f165c91c-099f-457c-8ba7-e9b93953116f]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:32.732136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:32.732136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:32.732136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:32.732136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:32.732136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:32.732136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:32.732136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:32.732136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:32.732136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:32.732136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:32.732136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:32.732136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:32.735743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f165c91c-099f-457c-8ba7-e9b93953116f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:32.736492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:32.739051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:32.744222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93b88a4e-a423-45c3-bf5d-33aef126231e]: (4, ['qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.744222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:32.744222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:32.744222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:32.744222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:32.744222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56e3b059-e10d-4636-8a87-518286f53af6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.745419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[014addad-d6ac-4eb6-90fb-af1cdf5cbcba]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.746838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a58db247-7089-4a16-97af-83e616efe817]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.768485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c0e1f01-b18a-423c-b5f7-8184590ee083]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 380862, 'reachable_time': 42916, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155738, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.771289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:32.771809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3db08645-83b1-4611-9f12-5f6c88e84b46]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:32.772807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:32.776853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 1, update_id 68fd2f7f-1ba4-4013-8f1a-930995f5f868. Wait time elapsed: 4.020 Jul 01 09:48:32.797704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 1, update_id 68fd2f7f-1ba4-4013-8f1a-930995f5f868. Wait time elapsed: 4.020 Jul 01 09:48:32.797704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:33.600514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.826s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:33.602297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:33.604127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:33.605225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:33.605775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:33.609080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42de90f2-ac9f-4293-8c1b-47846c783905]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.645407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97fa68c4-1cce-4064-8926-65b941a7c0ff]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.675721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6ee5842-c42c-4790-8d6e-0a1d75483f5b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.718729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1dfa809d-e84c-4c75-9f47-c7c1f688bcf7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155744, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.740741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba7f4fb0-8966-4c85-ad81-9e846915525b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.766858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6a260a9-81a1-4e95-8ccf-a49297a93349]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.792018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a548496-cfb1-4c43-a45a-30d1a1c77caa]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.816806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7afd682-8a17-46a2-ae1e-b1000afa5a27]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.817361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:33.817642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:33.818132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:33.819013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:33.840302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:33.841391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64ec9ff1-82f9-4a0a-814a-68ad7a397122]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.845573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5311e67-07df-4a64-957e-f5a4f659c2e9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.847515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:33.885856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:33.888724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:33.889696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:33.890551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:33.891437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:33.939253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b96a4f0-101c-4835-acaa-ac2447a61f35]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.982430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[25fc5b44-9e23-4189-b87d-38702852c235]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:33.982430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:33.982430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:33.982430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:33.982430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:33.982430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:33.982430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:33.982430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:33.982430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:33.982430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:33.982430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:33.982430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:33.982430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:33.983847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25fc5b44-9e23-4189-b87d-38702852c235]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:33.984939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:33.986875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:33.990573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[437d5e73-10b7-4216-b565-7d09ba20da1b]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.990573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:33.990573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:33.990573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:33.990573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:33.990573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc68170e-0c93-4f4c-87c0-9aa6c968d487]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.991180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[407e27ea-96bb-4f9c-ab6c-23cc7cb6b171]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:33.992971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[603dba03-3f00-4e36-9aa0-dc5cc77623df]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:34.013681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56d0ca6e-4650-4481-b3a6-6994c3f2a670]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 380998, 'reachable_time': 34540, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155762, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:34.022635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:34.023487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9ad1ea8-09e7-4efe-97bc-850ba239f748]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:34.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:34.033489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 169688c0-a1e6-41ac-a46d-80f52e392a1a. Wait time elapsed: 7.977 Jul 01 09:48:34.056198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 169688c0-a1e6-41ac-a46d-80f52e392a1a. Wait time elapsed: 7.977 Jul 01 09:48:34.056198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:36.102708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.072s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:36.103424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:36.103928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:36.104258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:36.104606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:36.106173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad28cc7e-0bdb-4d87-83cc-01de80cd1b33]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.120302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73adf23b-8f25-4a03-8354-6a492526c339]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.142131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b8b84fe-cc50-40b6-a9a4-4950033b78cc]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.170482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43f12929-28e4-4e9d-b809-0c227e284745]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155767, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.193395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1df68335-fc6d-4aad-ad68-91ccbdf0d041]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.217951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb868fdd-b2b5-4711-a0ba-35cfb501d76c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.239341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9fec19f2-7f37-4f0d-9713-4fc5ff8d5029]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.259667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f12c660-3340-42fa-b24c-366d66d9a60c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.260848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:36.261193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:36.261527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:36.261904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:36.278002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:36.278303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a350ecb1-8019-4d72-a41e-eb452960017d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.281015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f28478b-f94b-4ce3-8859-bc08c193838d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.282013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:36.323034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:36.323720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:36.325296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:36.325296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:36.325296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:36.375234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fede7309-74b8-4a45-b4f9-c5c3f545ebec]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.404893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[32b8b9f6-f75c-4bfc-a84e-19a6d0020fb5]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:36.404893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:36.404893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:36.404893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:36.404893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:36.404893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:36.404893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:36.404893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:36.404893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:36.404893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:36.404893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:36.404893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:36.404893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:36.406938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32b8b9f6-f75c-4bfc-a84e-19a6d0020fb5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:36.407987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:36.410540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:36.413360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[269b0760-d102-49e9-abb0-097b67ad25ec]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.414025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:36.414356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:36.414616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:36.414965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:36.416105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6a949b9-9b95-49b9-8997-342220c20f94]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.417144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb498045-cbe7-4214-9c0f-7bc90879d257]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.418221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c95ed0b4-7d50-4713-b0b3-cf72a2d418f5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.443628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8524e25-f095-4c8a-9c76-b2a96eeb1484]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 381243, 'reachable_time': 16736, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155786, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.451501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:36.452036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5603bba5-232d-402b-b2ef-847cf9f54712]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:36.453461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:34Z", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:36.459180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:36.475851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 68fd2f7f-1ba4-4013-8f1a-930995f5f868. Wait time elapsed: 7.705 Jul 01 09:48:36.475851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 68fd2f7f-1ba4-4013-8f1a-930995f5f868. Wait time elapsed: 7.705 Jul 01 09:48:36.475851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:37.365576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.905s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:37.366209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:37.366771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:37.367093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:37.367400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:37.368836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[736a942b-9c6a-471a-9d18-bfb211cf3f95]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.398750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b0f6bf0-27f1-41ed-ac83-1c229c69326d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.445037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12ef231b-057c-4706-b171-154ddd2a2262]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.489828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1aeac69e-3d1f-4ad9-8700-b09cdfcb4a14]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155831, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.523863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a64e4c6-5c71-4040-b873-96f689202c11]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.552500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b993581-45da-4eb0-b125-abe2176df270]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.581355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9704468-9a25-46e9-86cf-1c2692a98241]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.608899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a916e40-c119-43d2-8e9c-4a1179c8cd39]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.609511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:37.609782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:37.610046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:37.610322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:37.629416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:37.629705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b54f5dd6-c7b1-471e-8e52-429f78903f3b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.633206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ecfbd93-edc9-46b0-bc04-ef30cb0185dc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.636350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:37.686094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:37.688594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:37.689288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:37.689837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:37.690375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:37.732677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7559fae-0a5e-4e2a-b09a-daf989a6faf2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.771595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ecc94bea-21a9-49a8-8f7d-c5b3e5daf989]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:37.771595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:37.771595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:37.771595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:37.771595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:37.771595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:37.771595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:37.771595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:37.771595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:37.771595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:37.771595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:37.771595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:37.771595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:37.773083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecc94bea-21a9-49a8-8f7d-c5b3e5daf989]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:37.774144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:37.776327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:37.780124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[724d4413-bd31-49b3-be78-ede37b38acc1]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.780888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:37.781461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:37.781864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:37.782256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:37.783908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76620386-e9b8-4f1c-95f9-443b920024ca]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.784879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a16e88f3-7847-4d82-89af-6da7734903f9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.786504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26cc26f4-d38c-4315-b3c1-f5b1678fbe85]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.807626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[849f5a33-99c6-4181-99dd-188b845b10b0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 381375, 'reachable_time': 15380, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155853, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.809819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:37.810346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65abc4c9-c35a-47b9-88ed-4e941b055699]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:37.811660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:36Z", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:37.817318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:37.834125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 169688c0-a1e6-41ac-a46d-80f52e392a1a. Wait time elapsed: 11.764 Jul 01 09:48:37.834125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 169688c0-a1e6-41ac-a46d-80f52e392a1a. Wait time elapsed: 11.765 Jul 01 09:48:37.834125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:40.269141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.449s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:40.270390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:40.271948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:40.272276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:40.272559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:40.276409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77ec97de-50d5-4fad-937f-86d701bb74b4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.294866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e144c123-ae7c-45dd-8a3e-1f4f060894e1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.315644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[330b1705-b0c8-4d1e-87f3-3fef20723ced]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.341357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04b3950c-7227-41e2-92c7-a3a415a71176]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155859, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.381455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[daa67c51-fd9f-4eaa-a711-2a2897783861]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.401852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54b4481b-2f66-4c74-880e-6618b5d6e9b0]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.422574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49ab0a7a-7fcc-4dd6-abba-71fce6d74695]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.444236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a61a4ef7-6e2c-4320-bcaf-faa4e15ccf08]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.444802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:40.445411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:40.445883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:40.446160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:40.467644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:40.467942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2135df3-c3e7-42e6-8a27-8627b10959e0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.470615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa060bf6-1c3a-4117-81ec-df476345fe71]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.471599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:40.515344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:40.516085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:40.516478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:40.516870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:40.517251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:40.556115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a411b73-904b-4b61-b4fd-fc1126587acf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.589938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b5ac6255-60d4-4a29-9249-ef96ad113000]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:40.589938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:40.589938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:40.589938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:40.589938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:40.589938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:40.589938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:40.589938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:40.589938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:40.589938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:40.589938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:40.589938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:40.589938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:40.590956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5ac6255-60d4-4a29-9249-ef96ad113000]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:40.591140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:40.592858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:40.595059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eaa9b949-dbfe-410a-ba4a-918ff8b82888]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.595059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:40.595059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:40.595059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:40.595059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:40.596516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f676c98-8e5e-48aa-a29f-26c284bd730c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.596978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e68c87d3-8e1b-4bb4-9393-cae473978a0a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.598548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e093e24b-c49d-40a6-96c3-c4b52ab82be2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.617013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5b1920c-f092-42b4-ac7f-0d2f40417eba]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 381660, 'reachable_time': 36308, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155877, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f898a85-7d6f-4055-b394-c78ea17d1e1c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:40.621685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:38Z", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:40.625607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 68fd2f7f-1ba4-4013-8f1a-930995f5f868. Wait time elapsed: 11.868 Jul 01 09:48:40.639187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 68fd2f7f-1ba4-4013-8f1a-930995f5f868. Wait time elapsed: 11.868 Jul 01 09:48:40.639187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:41.406167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.783s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:41.406855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:41.407761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:41.408288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:41.408642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:41.410539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87b4c28f-7476-41d9-9005-670adb7fd484]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.567940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66e0232a-9e2e-4c38-8bf7-30b5119b5d6f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.589711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9e8200a-50f9-463e-924b-492745dd0181]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.618260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5144adfa-863f-46f6-9e48-ff58f3d6f937]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155880, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.642132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11c72057-ceb0-4a1e-bdbe-1c4c4fe3dcb0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.665320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6a429b0-fbd7-453f-a142-da22f98093ab]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.685335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ad074a1-fc21-4636-89f7-cb5f531c8606]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.706370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f4fd6db-6ad6-405a-a43d-8f28e01d39d4]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.708033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:41.708322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:41.708592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:41.708870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:41.723496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:41.723789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e77a4033-dbd8-4775-ad48-d6a1aadb4a13]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.726462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84ee9fd8-8d60-452b-bae6-415ce6a5052a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.727437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:41.764557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:41.765353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:41.765697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:41.766073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:41.766729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:41.811932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8fe5129d-0ef5-4572-baf8-8db06a29c9d8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.843793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a1a0b50e-ba18-4a61-a8d2-4e5495ce6589]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:41.843793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:41.843793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:41.843793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:41.843793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:41.843793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:41.843793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:41.843793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:41.843793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:41.843793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:41.843793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:41.843793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:41.843793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:41.844779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1a0b50e-ba18-4a61-a8d2-4e5495ce6589]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:41.845049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:41.849948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:41.854135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d056c76b-0779-40b1-8150-0d422f5848cc]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.855198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:41.855735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:41.856043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:41.856388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:41.857498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63c8d07c-7038-43da-9b31-047530b996c8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.858436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2136b811-b468-4f91-8240-405f6781fe3e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.859459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9eeeaddb-e066-4a9c-bb18-ac95f696a527]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.878184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dad52f9d-816e-4d2e-8f1a-4f900a353fae]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 381788, 'reachable_time': 27136, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155899, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.883448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:41.883820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3db0355d-fb5f-4577-9629-4133b7ee76dd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:41.885177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:40Z", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:41.888717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 169688c0-a1e6-41ac-a46d-80f52e392a1a. Wait time elapsed: 15.833 Jul 01 09:48:41.902122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 169688c0-a1e6-41ac-a46d-80f52e392a1a. Wait time elapsed: 15.833 Jul 01 09:48:41.902122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:44.850899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.964s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:44.851706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:44.852394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:44.852950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:44.853471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:44.854790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f5e9320-c23f-48ea-aa48-c416999b2028]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:44.868392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af18239c-3832-4de7-80a2-de80b46ed9d9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:44.889865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8035a420-8171-464e-9130-c5dea6257f2e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:44.921367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75ecb7b4-c250-4382-bbd3-906c143f9159]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155949, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:44.942821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0963b46f-5704-403e-a3f6-0220e16c82ad]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:44.963936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6474f048-e072-4140-83ba-7812b9eec900]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:44.986439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e60b48f3-cba1-4f8c-ab46-a4629880f53b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.012509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b62bccd-679e-48f7-9c3d-34679e35a480]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.013364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:45.013817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:45.014196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:45.014598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:45.034686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:45.034968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6182282-3d97-4fc1-8c36-883af317beb2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.037972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[886e86ef-b693-46d7-9ca6-6c29c91f71bf]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.038931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:45.082143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:45.083042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:45.083877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:45.084458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:45.084834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:45.137324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e252ae88-3947-4a95-9128-8703d7e5020b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.176218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4cfe2279-57c7-4b95-8d08-426d945a3cb4]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:45.176218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:45.176218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:45.176218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:45.176218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:45.176218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:45.176218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:45.176218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:45.176218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:45.176218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:45.176218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:45.176218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:45.176218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:45.177375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cfe2279-57c7-4b95-8d08-426d945a3cb4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:45.177533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:45.179355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:45.182871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b21ed125-8949-463b-bc05-59cbdbdd92a9]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.182871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:45.182871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:45.182871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:45.182871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:45.182871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f6a4b9a-f357-4534-bd5e-a468c5c45453]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.183571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c3c0776-4949-4c8c-8321-84fbc8567abf]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.184467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81c730ff-7b80-4bc2-b863-b4d28be67420]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.204287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c779702-e568-4ac2-8f84-d238bc63b239]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 382118, 'reachable_time': 29320, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155968, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.206294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:45.206566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40c57c3f-a7d2-4256-a446-151bba4ed748]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:45.207842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:42Z", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:45.212818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:45.228978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 68fd2f7f-1ba4-4013-8f1a-930995f5f868. Wait time elapsed: 16.459 Jul 01 09:48:45.228978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 68fd2f7f-1ba4-4013-8f1a-930995f5f868. Wait time elapsed: 16.459 Jul 01 09:48:45.228978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:45.917845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.704s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:45.918974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:45.919953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:45.920274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:45.920551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:45.921828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aeebfbcd-d6c2-4371-a6b0-c2f1d28cb5a8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.934871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c98115f-aba2-4206-a3f0-2a65bfe64555]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.955331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef5f1292-cf63-48b6-9a55-e2aebc40481f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:45.982123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c85542d-ba69-4e9d-be92-de0d8e58b573]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 155974, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.004771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2db27bfd-c0c4-4de8-88d0-bcff3f1cd534]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.025319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1136a3f8-448a-43d4-bf0c-6bb70612b954]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.053127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86b92f07-f0df-44c3-a2c1-b948f3a8c458]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.085223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e41c7533-3661-4a36-9776-9c272b7a3868]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.086065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:46.086429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:46.086886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:46.087529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:46.103931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:46.104223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd3140f6-5a6e-43b6-8ce8-595bffc11aba]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.107145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81d7368d-0a3d-4be2-9bc4-6cf7cfd2894a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.108446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:46.142031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:46.142946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:46.143651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:46.144561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:46.145273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:46.184170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[afa421bc-26cd-4c4f-9db6-65899b7d81be]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.224975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1105cd0c-2e8a-4af5-a154-e6ea56a59b38]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:46.224975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:46.224975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:46.224975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:46.224975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:46.224975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:46.224975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:46.224975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:46.224975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:46.224975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:46.224975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:46.224975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:46.224975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:46.226039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1105cd0c-2e8a-4af5-a154-e6ea56a59b38]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:46.226249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:46.228291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:46.230554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6fad9a6-f9ed-4a61-9c48-c33b99fcefbb]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.230554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:46.230554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:46.230554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:46.230554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:46.231005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19971949-c130-4377-a84c-bbd15b6b63b5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.231832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9981d16-a2d1-470f-a8d1-bbf7d5c21293]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.232812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24a7803d-c8f0-43d3-98f6-ce5617158817]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.260218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69068c2b-0c57-43aa-b3e3-249adcdbfeb9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 382224, 'reachable_time': 21004, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 155992, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.266757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:46.267098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0050a561-24d0-4437-a27f-54d31969775f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:46.268348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:45Z", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:46.271431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 169688c0-a1e6-41ac-a46d-80f52e392a1a. Wait time elapsed: 20.217 Jul 01 09:48:46.285617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 169688c0-a1e6-41ac-a46d-80f52e392a1a. Wait time elapsed: 20.217 Jul 01 09:48:46.285617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:48.078427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-898887fa-9bf4-4ab7-a1bb-aa2e5ce4416a tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['a6bfbb45-cc90-4764-a46d-cff9e14d2acc'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:48:49.133688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.862s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:49.134255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:49.135008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:49.135452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:49.135730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:49.136956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b30ab6c-208e-4a3a-ae53-50ae28157cca]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.149912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22146ca0-a345-4589-b3a6-cc056da82573]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.170682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52dba57f-3883-47fe-bd96-04e76a5cee4b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.199127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6c60782-1194-468f-93c2-45454cecdae7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 156042, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.220615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a08faac-1593-449c-ba0f-f90ef15c8b33]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.242205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f1a4cd0-6e8e-41a6-ae7b-f77b5d359094]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.262647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[527023b4-38b2-4bdd-951a-668f97a75258]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.288972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4bfbdba2-b0fa-4b0f-9fe6-ea7bbcd94e10]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.289537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:49.289793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:49.290048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:49.290651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:49.308643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:49.308944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f7e5fd0-6217-4a2b-bb4a-9a7b4c9b1c3f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.311870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f6b3917-7e8a-4a76-9c02-1d9cd037099b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.313062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:49.367594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:49.368279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:49.368791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:49.369260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:49.369930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:49.423291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5087628c-cfbd-43f1-bc68-7caea20fa73d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3deab13e-c525-4e1d-84bf-51e56a5b4640]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:49.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:49.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:49.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:49.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:49.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:49.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:49.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:49.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:49.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:49.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:49.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:49.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:49.467335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3deab13e-c525-4e1d-84bf-51e56a5b4640]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:49.468266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:49.470219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:49.473343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b658a77-3f74-4739-8e94-7b26f3a20d0a]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.473343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:49.473343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:49.473343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:49.473343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:49.473773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73b9874d-e961-4e9d-8749-ba83adc57182]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.475257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d59b3b1-9389-4975-aee4-651dcd960f97]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.476186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7b892c7-9777-4de7-836a-efa9b1b366c4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.498549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc551b1a-d1bd-4f94-8252-609e8d6f6249]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 382546, 'reachable_time': 41628, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 156061, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c776d2b0-04a2-4597-a577-bec0e60336fe]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:49.504600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fb5a4880-3748-43bb-9490-7b925f1babad", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.132", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4b4741c1-8539-4915-b85f-ae1834d083ca", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2774, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:4a:2f", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "2883534c-d88b-4aac-a116-9c6874f9599f", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:47Z", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:47Z", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:47Z", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:49.509503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3 Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 1, update_id 8ccb3616-ad18-4ed4-b691-6eb39d917ff8. Wait time elapsed: 1.426 Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 1, update_id 8ccb3616-ad18-4ed4-b691-6eb39d917ff8. Wait time elapsed: 1.427 Jul 01 09:48:49.531288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:50.329079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.822s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:50.329674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:50.330245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:50.330564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:50.330841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:50.332576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c7849be-4ed9-476f-a6e5-28643e2e49c8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.343214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b65e316-90f9-4143-9f9a-bc8cb6618fa2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.366120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66494c08-406b-4c35-bb7e-f819782d26aa]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.397958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88b78ccf-88b3-4ce8-a4c7-a5ee43067d2e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 156064, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.418807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81a36b0a-28a3-4078-8b04-9c1c30928ddd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.440362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb936dc5-c402-4989-9f3f-96d7aabc2191]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.461399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d26797cd-fc87-40c7-9ecc-c1c83c503139]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.480841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39c50710-265f-492d-a110-67e2429a0a78]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.481375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:50.481649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:50.481914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:50.482189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:50.498462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:50.498858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b692909b-525f-433b-af48-ec0e74d60482]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.501701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa5a7118-623c-4501-8dd8-ee7c089f66bd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.502734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:50.538223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:50.539261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:50.540857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:50.541323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:50.541927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:50.580506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48678ce6-843a-4bf3-a813-398ee4ddddf6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.609416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cb2bc0a9-8238-473e-9529-dd55af9660d6]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:50.609416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:50.609416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:50.609416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:50.609416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:50.609416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:50.609416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:50.609416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:50.609416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:50.609416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:50.609416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:50.609416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:50.609416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb2bc0a9-8238-473e-9529-dd55af9660d6]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:50.610619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:50.612792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:50.615708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9750fe1c-5bc8-43aa-9a88-2436e0cdc39e]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.615708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:50.615708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:50.615708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:50.615708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:50.616652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c47a73e-d4ee-49dc-b1d7-5ed747cd7892]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.617621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1483cf69-8237-4d64-86bb-ccfd6c9a4293]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.618481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49659e40-5ecf-4cef-a63b-5ef89684be72]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.730805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[067a202a-0331-4fc9-b577-b71e78a84b9c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 382666, 'reachable_time': 15064, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 156083, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.737293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:50.737669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2211f384-a80a-4b78-ad6c-af6fe9c8fb53]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:50.738763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fb5a4880-3748-43bb-9490-7b925f1babad", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.132", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4b4741c1-8539-4915-b85f-ae1834d083ca", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2774, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:4a:2f", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "2883534c-d88b-4aac-a116-9c6874f9599f", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:47Z", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:47Z", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:50.742206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:49Z", Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 68fd2f7f-1ba4-4013-8f1a-930995f5f868. Wait time elapsed: 21.986 Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 68fd2f7f-1ba4-4013-8f1a-930995f5f868. Wait time elapsed: 21.986 Jul 01 09:48:50.757316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:53.551301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.811s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:53.551935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:53.552789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:53.553038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:53.553344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:53.554466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64f74cd8-82c6-46b6-835c-ca3f9a50adc1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.569488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4299730a-3786-4dc0-b805-a3e44f9ca499]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.589920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf79701e-dacc-46ad-b2a1-df5ba8468cfe]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.616289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3ececa9-0cc7-4eb3-ad6b-1ee7daeb4170]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 156133, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.636529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a39f0e5a-5b52-4321-9641-6f628fe26675]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.657063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a154065-774e-4ab3-adf7-21e3b3281b58]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.676725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[418bcbe7-c85e-47dd-bf3e-c9087a6cad8c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.697270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f308b153-07ba-4ffe-b611-fa5cb250a021]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.698048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:53.698328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:53.698618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:53.698902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:53.720929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:53.721236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1c6d591-bf58-4668-8cb2-7db7734f8f36]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.723905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1560ea8-0013-447c-b1ac-8f0ee7457537]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.725007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:53.754629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:53.756118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:53.757522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:53.758346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:53.758752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:53.800802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ea669d8-e366-45a0-8300-484854dcd847]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.838418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[515b5155-d923-487f-a6cf-a8fca84fb5eb]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:53.838418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:53.838418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:53.838418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:53.838418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:53.838418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:53.838418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:53.838418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:53.838418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:53.838418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:53.838418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:53.838418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:53.838418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:53.839559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[515b5155-d923-487f-a6cf-a8fca84fb5eb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:53.839787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:53.842633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:53.845101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ddad6c9-b4ff-4da7-a919-5a9b616e321b]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.845101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:53.845101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:53.845404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:53.845610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:53.846714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52d2d5c0-0173-43aa-b84f-05915abc21e4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.848419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a07e82a6-0927-42d8-b0d7-c51377853edc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.849324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81b123ed-967e-43d1-a9ca-d29c34592ec5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.867518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f3491ba-6fff-44c6-a6b7-a1416a397f3f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 382988, 'reachable_time': 38260, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 156151, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.874084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:53.874364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92720c6b-990f-43f2-bf48-566c51b2783f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:53.875478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fb5a4880-3748-43bb-9490-7b925f1babad", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.132", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4b4741c1-8539-4915-b85f-ae1834d083ca", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2774, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:4a:2f", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "2883534c-d88b-4aac-a116-9c6874f9599f", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:47Z", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:47Z", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:53.879206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:51Z", Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 8ccb3616-ad18-4ed4-b691-6eb39d917ff8. Wait time elapsed: 5.797 Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 8ccb3616-ad18-4ed4-b691-6eb39d917ff8. Wait time elapsed: 5.797 Jul 01 09:48:53.891401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:54.629511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.752s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:54.630148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:54.631932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:54.632322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:54.632806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:54.634180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a23591c-7f4b-4e91-8b10-885b5d348c11]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.652564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d34ef5c-cedc-4ea8-9396-9fef1958f821]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.674414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68fc60e6-f0b6-4e6b-abb3-7b244299659d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.703689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[337bbbbd-66fc-4a62-aa31-237c20cc2bdc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 156155, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.724434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c49081ed-c3a9-418c-a5e0-399173be46fe]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.745918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b481b00a-6a07-4ae1-bc0a-8955cd8fe2b5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.766105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea4306fa-07dc-42ef-b60c-f41054e2e60a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.788566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d19d8a9e-7e4a-43ec-9dab-ac46222a8f15]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.789483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:54.789971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:54.790449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:54.790945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:54.809195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:54.809479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12af62c5-b3fe-49cd-9d65-d7a8b60eb6ed]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.812548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9df04e0-9870-4993-8270-3da928db1c2d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.814215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:54.838500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:54.839619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:54.840215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:54.840829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:54.841209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:54.890259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb907ef1-6cb5-4dc3-99d7-fd209c17320f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.919773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6f8402be-dbb2-4666-b6ef-9fa069380267]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:54.919773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:54.919773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:54.919773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:54.919773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:54.919773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:54.919773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:54.919773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:54.919773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:54.919773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:54.919773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:54.919773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:54.919773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:54.920956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f8402be-dbb2-4666-b6ef-9fa069380267]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:54.921063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:54.922920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:54.925776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ede50ad-cd03-449a-9b21-cc53fbb2435e]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.925776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:54.925776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:54.925776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:54.925776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:54.926762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2de79ca4-5777-4c9a-89df-0631d14821d7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.927658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[137e3c3b-b64f-49b4-b891-5e4340a0eb76]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.928671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e1e46a7-094d-4d01-9366-797c1c931c5c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.946550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d81095c-26bd-441f-b69a-2209602d5e27]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 383096, 'reachable_time': 16616, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 156173, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.952424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:54.952748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[250eba60-d8c5-4391-84c8-4e0da1145a34]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:54.953872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fb5a4880-3748-43bb-9490-7b925f1babad", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.132", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4b4741c1-8539-4915-b85f-ae1834d083ca", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2774, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:4a:2f", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "2883534c-d88b-4aac-a116-9c6874f9599f", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:47Z", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:47Z", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:54.957788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:53Z", Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 68fd2f7f-1ba4-4013-8f1a-930995f5f868. Wait time elapsed: 26.201 Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 68fd2f7f-1ba4-4013-8f1a-930995f5f868. Wait time elapsed: 26.201 Jul 01 09:48:54.970186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:55.154054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:48:57.958111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.002s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:57.958733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:57.959870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:57.960206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:57.960487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:57.961687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb538a81-fe2b-4545-a4ef-dc8493e011cb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:57.976726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26f77995-4d98-4850-b134-ceaad73c0a08]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.001573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74f2c5d2-5b54-4bda-ba21-2f3ce370bdc1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.033221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bfa7c41-6eec-4182-9d89-ac48c049aef4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 156227, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.057045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9a7f08f-bdd7-4a6f-9619-2a0438a02b48]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.068546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6b946b0-f73c-42a1-912f-de6ddd2b7c3e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.103628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71507945-6573-4ec7-a890-c8a813e2c0e0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.142243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b75070e-45d4-491f-a626-ae19eb80f6dc]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.143164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:58.143682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:58.145204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:58.145718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:58.171534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:58.172187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1ca0f9e-dee6-4781-9f55-8d180e8978a8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.176009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05f9a4ca-4795-4c3b-92dc-20d42481919e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.177222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:58.214637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:58.215724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:58.216658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:58.217221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:58.219172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:58.266928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab55daf7-dbe1-4c46-b2fa-4bba8f3d7a32]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.307665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[35581ad5-5710-4119-bd53-74aa54d5df56]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:58.307665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:58.307665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:58.307665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:58.307665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:58.307665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:58.307665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:58.307665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:58.307665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:58.307665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:58.307665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:58.307665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:58.307665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:58.309773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35581ad5-5710-4119-bd53-74aa54d5df56]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:58.310777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:58.312746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:58.325489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59603a4d-73d3-4d82-a065-48372ad0b1d0]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.326295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:58.326615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:58.326869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:58.327503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:58.328927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1837bbb9-bb7e-4879-8cae-f57f4f25c21b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.330001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b86aacb-8b2d-4a8d-9c04-2d92ffe5b03a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.330924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8903c897-1ab3-4602-8b72-7e4b44361702]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.353025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[951f5d4f-a2c3-42fb-a83a-9edb935c6e11]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 383429, 'reachable_time': 43976, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 156247, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.355330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:58.355937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cfaf3700-4c54-4a43-82ee-4dc3c016c583]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:58.357154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fb5a4880-3748-43bb-9490-7b925f1babad", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.132", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4b4741c1-8539-4915-b85f-ae1834d083ca", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2774, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:4a:2f", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "2883534c-d88b-4aac-a116-9c6874f9599f", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:47Z", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:47Z", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:58.363422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:56Z", Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 22, Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3 Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 8ccb3616-ad18-4ed4-b691-6eb39d917ff8. Wait time elapsed: 10.285 Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 8ccb3616-ad18-4ed4-b691-6eb39d917ff8. Wait time elapsed: 10.285 Jul 01 09:48:58.377046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:48:59.372229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.007s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:48:59.372823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:48:59.373563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:48:59.373946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:59.374304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:59.375512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3411a239-4df2-46e4-9868-16d3409e071d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.386284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40945245-8c42-475b-a682-3add4914bb33]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.410615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[644b32db-0de7-4eb4-8a62-734ddb60b1db]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.446587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28f5e7ff-a6c7-4509-9530-34696de39132]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 156253, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.467565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c82906a-b3ca-419b-a64b-44dcc15350ad]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.490321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7191b2b5-ff12-4e02-ad32-8431b42d4e6f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.510899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46e819ec-9475-418b-84eb-46f4afc52d03]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.531182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6f67e9b-8c00-499d-86cb-f121324da7a0]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.531766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:48:59.532040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:48:59.532307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:48:59.532586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:48:59.547298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:48:59.547712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23dba811-08fc-4e26-8c96-9b6eee63ff9d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.552586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[757a4179-2022-42f1-906c-39a88ffd8567]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.553603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:59.593893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:59.594704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:59.595305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:59.596078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:59.596504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:48:59.640891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00ab73c9-c0fb-4996-a31c-a5544bf15307]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.669518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7ee7a560-72b0-4e8d-b72e-d284d824f54b]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:48:59.669518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:48:59.669518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:48:59.669518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:48:59.669518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:48:59.669518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:48:59.669518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:48:59.669518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:48:59.669518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:48:59.669518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:48:59.669518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:48:59.669518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:48:59.669518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ee7a560-72b0-4e8d-b72e-d284d824f54b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:59.670693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:59.672805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:59.676350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65d17658-95a4-4cff-98c8-2b77d0f5a23f]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.676350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:59.676350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:59.676350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:48:59.676350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:48:59.677228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d90fa75-b2e5-4302-9c3a-650051bdc642]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.678262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20b260a5-f149-41d6-ac9e-c0299a2d2aef]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.679239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e951204-74e6-4746-a65e-ab93672949e4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.698087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f1bdc23-d9b5-4d4e-bc94-d71ee423b2dc]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 383570, 'reachable_time': 29772, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 156272, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2353355-6914-453f-8258-f28bd03eb08e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:59.711086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fb5a4880-3748-43bb-9490-7b925f1babad", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.132", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4b4741c1-8539-4915-b85f-ae1834d083ca", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2774, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:4a:2f", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "2883534c-d88b-4aac-a116-9c6874f9599f", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:47Z", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:47Z", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:57Z", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:48:59.716502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 8ccb3616-ad18-4ed4-b691-6eb39d917ff8. Wait time elapsed: 11.626 Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 8ccb3616-ad18-4ed4-b691-6eb39d917ff8. Wait time elapsed: 11.626 Jul 01 09:48:59.732233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:49:02.638954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.933s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:49:02.640115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:49:02.641277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:49:02.641694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:49:02.642784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:49:02.644288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb746fe9-2bf8-4b34-8634-97c52db51bca]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.660306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a14891fc-5c04-4aa7-9e31-80caabd3c10b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.681312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a1c9e92-0988-4278-ab19-bb3781f37b15]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.710472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85d81da0-bcad-49da-b599-61dfa3e50f0d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 156322, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.731536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ceb11b8-ace1-4511-8278-9695c24cb216]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.768090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[856f7bde-b10e-4f8e-9dbb-2bb2ed9c9f51]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.792173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f1b349c-bb8b-49c7-b968-7446fc727701]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.816501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eac458ce-5322-4a45-a67f-a30457c30032]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.817187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:49:02.817527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:49:02.817862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:49:02.818211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:49:02.833482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:49:02.833869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89b9b4ad-e3ca-4cce-84b3-7dfe10e20fa0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.837030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0eb9a1fa-9cc2-4916-acd6-78691e145b33]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.838093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:02.877795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:02.879334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:02.879967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:02.880363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:02.880748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:02.926926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[941b1c87-ad3e-4847-b11f-d3c22a8d752f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.962648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[be0f5b9c-db86-4d47-9df7-eaa1a575a56e]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:49:02.962648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:49:02.962648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:49:02.962648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:49:02.962648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:49:02.962648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:49:02.962648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:49:02.962648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:49:02.962648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:49:02.962648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:49:02.962648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:49:02.962648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:49:02.962648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:02.963909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be0f5b9c-db86-4d47-9df7-eaa1a575a56e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:02.964576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:02.966374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:49:02.969015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83f61f82-27a6-41df-9539-eb2890ccc4f4]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.969015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:49:02.969015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:49:02.969015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:49:02.969015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:49:02.969893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[faeaeb70-0159-46f1-a44a-fb482dce9517]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.970840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f6e31bb-5657-415d-bc0b-300457473463]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.971803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c67e9c4-a690-457f-961f-5745486f24f8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.989139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[067831a2-c0e9-4b19-902c-d81f085c16ff]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 383897, 'reachable_time': 37716, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 156341, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.994706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:49:02.995670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8aa41f81-2e38-4a44-9ec3-8b70ab118709]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:02.996849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fb5a4880-3748-43bb-9490-7b925f1babad", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.132", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4b4741c1-8539-4915-b85f-ae1834d083ca", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2774, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:4a:2f", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "2883534c-d88b-4aac-a116-9c6874f9599f", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:47Z", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:47Z", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:49:03.000109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:49:00Z", Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 24, Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 8ccb3616-ad18-4ed4-b691-6eb39d917ff8. Wait time elapsed: 14.919 Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 8ccb3616-ad18-4ed4-b691-6eb39d917ff8. Wait time elapsed: 14.919 Jul 01 09:49:03.021583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:49:03.735687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.736s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:49:03.736245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:49:03.736788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:49:03.737108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:49:03.737484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:49:03.739791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b1f332b-20f9-4a8d-a24a-d0060b1ea286]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:03.756126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5695abdd-eace-49b7-affc-b688bcabf1d1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:03.776448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[638e5653-c677-47b4-ba2e-07b036e62b88]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:03.803539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf316c8d-6570-438e-8014-6b59e7fc6180]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 156344, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:03.824051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ccf2d97-4ccd-4dd2-ae48-843de4770382]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:03.844302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38fc8a8f-3b5d-4501-a027-abeeb5fd757a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:03.864858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb331f5e-0b76-42de-95c6-424bc67099f7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:03.884730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d59b2a07-0fd1-4cc5-b5cf-85bd1bba3ab3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:03.885269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:49:03.885548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:49:03.885819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:49:03.886116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:49:03.901740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:49:03.902035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fae79b42-d3fb-426d-ad62-4917e17fd914]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:03.905337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a68409e4-6cdc-42c2-87cf-3b790cba23af]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:03.906417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:03.933241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:03.934054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:03.934629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:03.935191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:03.935645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:03.994320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6aa38ca-e2f0-42bd-a436-7cc0531ef8dc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:04.029373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e39a078a-e435-41c1-be4f-ca1dd4883209]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:49:04.029373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:49:04.029373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:49:04.029373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:49:04.029373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:49:04.029373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:49:04.029373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:49:04.029373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:49:04.029373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:49:04.029373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:49:04.029373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:49:04.029373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:49:04.029373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:04.030831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e39a078a-e435-41c1-be4f-ca1dd4883209]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:04.031461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:04.033411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:49:04.036037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[faef6f8e-3d48-4e2d-907d-58d9270bcca9]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:04.036037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:49:04.036037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:49:04.036354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:49:04.036675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:49:04.037786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96bca8af-bfb6-4d37-a690-347ec355a311]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:04.038730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28614cba-4c38-483d-85d9-fe9f991a397b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:04.039812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4ba5a38-2049-4359-8c5c-cefbb09e32bd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:04.057148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a07fd4f-476d-4cf1-80d1-d63bc34c4aaf]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 384006, 'reachable_time': 29640, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 156362, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:04.059039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:49:04.059318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[431385a6-12d6-404c-bc7f-6eef6e6cb091]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:04.060408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fb5a4880-3748-43bb-9490-7b925f1babad", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.132", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4b4741c1-8539-4915-b85f-ae1834d083ca", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2774, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:4a:2f", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "2883534c-d88b-4aac-a116-9c6874f9599f", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:47Z", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:47Z", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:49:04.063399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:49:02Z", Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 25, Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 8ccb3616-ad18-4ed4-b691-6eb39d917ff8. Wait time elapsed: 15.982 Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id 8ccb3616-ad18-4ed4-b691-6eb39d917ff8. Wait time elapsed: 15.982 Jul 01 09:49:04.078774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:49:07.084642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.022s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:49:07.085213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:49:07.085811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:49:07.086134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:49:07.086418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:49:07.087599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3993919-6adf-40f9-af9e-6f67c2617fcd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.101025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbb0d6bb-8799-4915-8d09-3eb93c72f71d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.129477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17a1226e-3740-4b09-9b75-d91f49e2c59a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.158770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b01185d-c7c6-43af-a212-13a207eb316d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 156414, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.180308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2301ee78-aa55-46e3-8c1f-7a800fa517a8]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.200548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee6fb89b-262b-4ee1-8c18-e6c2dc0c9fc1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.221569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec08e79c-12ec-453d-97c6-87ee8c95b270]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.254153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ffafefc-e427-4f8c-bcf0-027596d3eb06]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.254714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:49:07.254988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:49:07.255249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:49:07.255549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:49:07.276459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:49:07.276742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d251905-8161-4aaa-8f94-69ae67f0c65e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.280328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7774e540-9350-45b3-bccd-25d9d49413eb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.281272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:07.327282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:07.329166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:07.329752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:07.330682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:07.331888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:49:07.380800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b879ec0f-8457-485a-a4da-2543baf91615]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.419985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2a5d0b63-f4e8-4cd3-99df-b97a255589c5]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:49:07.419985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:49:07.419985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:49:07.419985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:49:07.419985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:49:07.419985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:49:07.419985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:49:07.419985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:49:07.419985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:49:07.419985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:49:07.419985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:49:07.419985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:49:07.419985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:07.421134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a5d0b63-f4e8-4cd3-99df-b97a255589c5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:07.422457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:07.424414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:49:07.428771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9a1074a-1fb6-4bca-8069-a19f4f1c9749]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.428771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:49:07.428771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:49:07.428771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:49:07.429297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:49:07.430467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[742b0391-bda2-4a69-8407-98027ce8d457]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.431471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a17497e-cbac-47b5-901c-8a6c947c59f6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.432605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc5d1e56-fe3c-40f4-9229-7f568ed946ef]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.452416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0bd25f96-4e41-40b9-b939-3af81f79df76]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 384342, 'reachable_time': 24524, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 156433, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8084d0d-bcb5-4fe5-9231-9e01a5896575]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:49:07.457941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fb5a4880-3748-43bb-9490-7b925f1babad", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.132", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4b4741c1-8539-4915-b85f-ae1834d083ca", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2774, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:4a:2f", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "2883534c-d88b-4aac-a116-9c6874f9599f", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:47Z", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:47Z", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:49:07.462179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:49:05Z", Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 26, Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:49:07.476544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3 Jul 01 09:49:15.096327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:49:15.097589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:49:15.097589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:49:15.097589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:49:35.156143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:50:15.100246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:50:15.100872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:50:15.100872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:50:15.101068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:50:15.158126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:50:55.160133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:51:15.098264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:51:15.098962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:51:15.098962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:51:15.099310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:51:35.163476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:52:15.099048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:52:15.099700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:52:15.099700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:52:15.099987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:52:15.165594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:52:55.168889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:53:15.103173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:53:15.103173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:53:15.103173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:53:15.103173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:53:35.169908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:53:49.578454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-510172b2-c655-4dff-863d-a5f9b95735dc tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['a6bfbb45-cc90-4764-a46d-cff9e14d2acc'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:53:49.579449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 1, update_id e6dea759-fab5-48ea-8659-c3e9149e32a9. Wait time elapsed: 0.001 Jul 01 09:53:49.579704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 1, update_id e6dea759-fab5-48ea-8659-c3e9149e32a9. Wait time elapsed: 0.001 Jul 01 09:53:49.579998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:53:50.331438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.751s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:53:50.331712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:53:50.332643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:53:50.332975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:53:50.333471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:53:50.334991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5178477a-20a4-4f40-98d4-6df3e406b2dd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.352069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c440e37c-41b3-4b3c-a099-222e1b10c5f5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.373009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a980fc96-dec7-4609-87a6-7fd840ecda8f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.403919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[750e61ef-22b6-4947-9f8f-ff80b64d2914]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 160899, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.426481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[355e61cd-18c0-4d6a-a8df-5261c0ff3e85]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.461686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02a69e6a-2c2e-4de3-9946-15725a1cb86c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.485046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ba873e5-3c2b-40da-9467-38263a008b94]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.507087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fcaac33-efd9-4826-b50a-6979889ab2e8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.507723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:53:50.508003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:53:50.508276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:53:50.508617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:53:50.524662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:53:50.525477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70bfb8ae-3b3f-45b5-bdd3-a69ebc30f001]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.530947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38af85ac-58ec-4c5a-8c68-9ab5604d6f08]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.532045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:50.565079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:50.566184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:50.566780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:50.567496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:50.568092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:50.633551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96ef2a43-db0e-40ce-8c20-0ccfaeeae8fc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.674609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c6ead4c1-85ea-4058-9705-60282dd11efe]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:53:50.674609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:53:50.674609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:53:50.674609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:53:50.674609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:53:50.674609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:53:50.674609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:53:50.674609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:53:50.674609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:53:50.674609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:53:50.674609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:53:50.674609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:53:50.674609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6ead4c1-85ea-4058-9705-60282dd11efe]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:50.675811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:50.679163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:53:50.684173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45b89975-2533-41b7-a898-1d7d2bb397b0]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.684173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:53:50.684173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:53:50.684173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:53:50.684173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:53:50.684173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe3298b9-1883-471c-b445-bd99f5dd85e4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.686787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ead15ae0-38c1-44fd-b4cc-9bd8c8961fe3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.687332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15eb7084-8433-4b40-b23f-536e6dc3130e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.712174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdce8531-7a16-4525-a218-5bfc0762135a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 412666, 'reachable_time': 44044, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 160918, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.714557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:53:50.715877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fff75d51-4da6-470f-bfa1-c16c86373b14]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:50.719068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:49:10Z", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:53:50.722257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id e6dea759-fab5-48ea-8659-c3e9149e32a9. Wait time elapsed: 1.142 Jul 01 09:53:50.739616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id e6dea759-fab5-48ea-8659-c3e9149e32a9. Wait time elapsed: 1.142 Jul 01 09:53:50.739616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:53:53.038596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.317s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:53:53.039163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:53:53.040459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:53:53.041297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:53:53.041592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:53:53.044741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7c0f570-340b-4491-8ed0-aed8b536dc5e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.060874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1413fc2f-b514-4fe2-88a4-c5ea0911b64a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.086111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38bcb30e-fce2-4ae8-87c0-2772e1845881]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.124992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c036b0b-5bac-4b0e-a223-b3e58e2b0257]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 160971, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.148777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[148fa6d8-8f2c-4dc2-aa13-b2e4f52a8475]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.171248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84578f29-0422-4bb9-9a53-e819d950113b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.192745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb4e53d9-5537-4e6e-97dc-22f975490ec6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.216630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6d61387-8c55-451f-aa71-b37d62ccf095]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.217516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:53:53.217799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:53:53.218074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:53:53.218353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(1cb279cd-4639-46e9-86ea-f3f2a1f3be91) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:53:53.235717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:53:53.236156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7294e649-d578-47cc-bb7c-8f7f25bc86da]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.239653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8af43a5b-7430-46b9-8056-a378fd98a079]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.240928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-1cb279cd-46, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:53.281156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-1cb279cd-46, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:53.282087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:53.282607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:53.283150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-1cb279cd-46, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:53.283717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-1cb279cd-46, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '1cb279cd-4639-46e9-86ea-f3f2a1f3be91', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f9:01:56'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:53.326332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33dec3e7-fe1b-4f8e-a71a-28e4eaf7cb3a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.373992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[40111ce4-1876-48c6-bed7-cd802c75161a]: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:53:53.373992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:53:53.373992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:53:53.373992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:53:53.373992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:53:53.373992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:53:53.373992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:53:53.373992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:53:53.373992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:53:53.373992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:53:53.373992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:53:53.373992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:53:53.373992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:53.374681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40111ce4-1876-48c6-bed7-cd802c75161a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:53.375192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:53.376428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:53:53.377908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c25df463-052b-43da-ac95-2ee8d248cb39]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.378105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:53:53.378661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:53:53.379058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:53:53.379574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:53:53.380595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14eb399f-a32d-41ad-9495-7501f319c07e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.381599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3b9fe5b-052e-4ac4-8009-89caa94bce01]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.382464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d73f914-e9c1-4f1e-8ad3-a5220127be71]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.406490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[794b6dfd-f632-41d6-a3ec-a386e1e9b759]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 412938, 'reachable_time': 23328, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 160989, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.413951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:53:53.414232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea1c77c0-c251-40ed-920d-32cd2059ce23]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-1cb279cd-46 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:53.415242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:27Z", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1cb279cd-4639-46e9-86ea-f3f2a1f3be91", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "f8fd78c5-9fd8-47d6-a985-6a235f840851", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f9:01:56", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2750, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:27Z", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:49:10Z", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "00478d63-39fe-4e9a-a54b-c62f51456d9e", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:53:53.419195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:53:53.429706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id e6dea759-fab5-48ea-8659-c3e9149e32a9. Wait time elapsed: 3.851 Jul 01 09:53:53.430089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 2, update_id e6dea759-fab5-48ea-8659-c3e9149e32a9. Wait time elapsed: 3.851 Jul 01 09:53:53.430513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:53:54.284606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.854s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:53:54.285173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a6bfbb45-cc90-4764-a46d-cff9e14d2acc info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:53:54.285935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a6bfbb45-cc90-4764-a46d-cff9e14d2acc), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:53:54.286350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:53:54.286690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:53:54.288585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a04fad98-8434-445c-bce2-e1c5870b2eea]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.311879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b82bb1c-2268-4c08-a754-8babf1bb8646]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.341513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ceed6a13-4c0f-49da-99eb-6ccd2ecd8098]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.375206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76abbb8a-858c-497f-8ce4-a21159636b54]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 160995, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.399083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[816cd9a8-6497-41b2-959d-c32386d6db56]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.424626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91245a76-2e3a-401a-b4cc-2789cb0da27a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.450316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28102b44-0d13-4b5f-acf3-7475e4eb1397]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.473529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8d6f158-9b06-4aff-9a49-f1a14c63f854]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.474397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:53:54.474777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:53:54.475135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:53:54.496232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[717a83dd-8466-44de-8f96-dad9af805ecb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 413063, 'reachable_time': 21368, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161001, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.497525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:53:54.497873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:53:54.498419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:53:54.498764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:53:54.499161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:53:54.499508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:53:54.499860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'f9aeffe6-476c-468a-844a-926e714fc4c1', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:b3:9f:b7', 'admin_state_up': True, 'status': 'ACTIVE', 'device_id': 'a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'device_owner': 'network:router_gateway', 'standard_attr_id': 2746, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.164', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:48:24Z', 'updated_at': '2022-07-01T09:48:30Z', 'revision_number': 4, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-f9aeffe6-47), ns(qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:53:54.516785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:53:54.517074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59594de6-5df4-425a-a45d-2574528a3c43]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.520494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f77a9d5c-2503-439e-b354-6530e47c3488]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.521597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-f9aeffe6-47, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:54.559395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-f9aeffe6-47, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:54.560182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-f9aeffe6-47, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:54.560650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-f9aeffe6-47, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:54.561112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-f9aeffe6-47, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:54.561595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-f9aeffe6-47, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'f9aeffe6-476c-468a-844a-926e714fc4c1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:b3:9f:b7'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:53:54.611930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d2ab7d0-e927-49a0-899e-bd0249d7b099]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.643485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bfe70e42-4f9b-40a3-9ab7-cc085b0e2fe8]: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:53:54.643485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:53:54.643485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:53:54.643485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:53:54.643485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:53:54.643485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:53:54.643485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:53:54.643485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:53:54.643485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:53:54.643485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:53:54.643485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:53:54.643485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:53:54.643485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:54.644267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfe70e42-4f9b-40a3-9ab7-cc085b0e2fe8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.644267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:53:54.644784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:53:54.668238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c65880e-efcc-497e-82a0-c2d921dbda58]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.687016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-2e27a5a4-40e2-41b7-9818-7ad087f3de22 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['a6bfbb45-cc90-4764-a46d-cff9e14d2acc'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:53:54.862694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0072977-1cc9-4d33-a6c5-a4e9f531732d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:54.878403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4293d80d-da79-4c73-a59f-3d9dfe99483b]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:55.046698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9d8d7cf-face-44d1-8a22-504d5c15de37]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:55.047582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:53:55.048090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:53:55.048565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:53:55.048880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc" released by "process_external" :: held 0.550s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:55.049667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:55.051283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:53:55.053206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9cddce6-823f-4f2e-b872-6a9015a2918c]: (4, ['qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:55.053206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:53:55.053206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:53:55.053206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:53:55.053456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a6bfbb45-cc90-4764-a46d-cff9e14d2acc.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:53:55.055887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4816288-1fd9-4c3c-b15f-954f5efcb405]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:55.057288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74b12f4e-bbae-49d5-8a63-d13f95e7469a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:55.059310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0db2ee89-acba-4ca2-8fdd-1f803aa5082c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:55.080780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[801bee35-6e1c-4f84-8cf1-f5ad74694f41]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 413063, 'reachable_time': 21368, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161020, 'error': None, 'target': 'qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:55.086422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f6e9dce-ee61-4e38-85f1-95057f6389ce]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a6bfbb45-cc90-4764-a46d-cff9e14d2acc: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-f9aeffe6-47 not found in namespace qrouter-a6bfbb45-cc90-4764-a46d-cff9e14d2acc. Jul 01 09:53:55.087703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-167062272", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164" Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:53:53Z", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f9aeffe6-476c-468a-844a-926e714fc4c1", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b3:9f:b7", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a6bfbb45-cc90-4764-a46d-cff9e14d2acc", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 2746, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.164", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:48:24Z", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:48:30Z", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 1, update_id f4b234a9-1c3b-40b3-9f29-5860cfebd4d1. Wait time elapsed: 0.401 Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 3, priority 1, update_id f4b234a9-1c3b-40b3-9f29-5860cfebd4d1. Wait time elapsed: 0.402 Jul 01 09:53:55.090784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:53:55.920638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-4d51828b-e2db-4837-b2ed-71c57c99924c tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got router deleted notification for a6bfbb45-cc90-4764-a46d-cff9e14d2acc {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:53:56.159898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.070s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:53:56.160696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router a6bfbb45-cc90-4764-a46d-cff9e14d2acc was not found. Performing router cleanup Jul 01 09:53:56.169881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c8bc0f2-30f0-40eb-afdb-d44e90b404f1]: (4, ['qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:56.170975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:53:56.171480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:53:56.171933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, update_id f4b234a9-1c3b-40b3-9f29-5860cfebd4d1. Time elapsed: 1.083 Jul 01 09:53:56.172384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 1, priority 1, update_id 53df6f4e-7064-4f09-99b2-69e4fd9a0a2e. Wait time elapsed: 0.251 Jul 01 09:53:56.172814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, action 1, priority 1, update_id 53df6f4e-7064-4f09-99b2-69e4fd9a0a2e. Wait time elapsed: 0.251 Jul 01 09:53:56.173268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router a6bfbb45-cc90-4764-a46d-cff9e14d2acc was not found. Performing router cleanup Jul 01 09:53:56.174911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f7e72b6-e743-489d-bd22-8901acdb669f]: (4, ['qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:53:56.176651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:53:56.179779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:53:56.181126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for a6bfbb45-cc90-4764-a46d-cff9e14d2acc, update_id 53df6f4e-7064-4f09-99b2-69e4fd9a0a2e. Time elapsed: 0.008 Jul 01 09:53:58.954706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-099eb935-1db9-4777-ae2e-f91792978aef tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['074757da-387c-402b-9869-921d663de888'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:53:58.961397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 1, update_id dae8712b-79d9-46a8-9666-415e4e322611. Wait time elapsed: 0.003 Jul 01 09:53:58.962041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 1, update_id dae8712b-79d9-46a8-9666-415e4e322611. Wait time elapsed: 0.003 Jul 01 09:53:58.965639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:00.360230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.396s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:00.362252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:00.363375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:00.363706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:00.364202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:00.366174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[889a5dbf-8dae-4f87-abb5-d25ced7582ed]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.385119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e9769eb-a050-4c1c-bccc-d933e8fa111c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.409820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f677ce86-cf58-4e0c-93e0-76ea78588e61]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.450005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f84e925-d4d6-42e0-a36a-854c3be7305f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161095, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.472584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1cc09682-23aa-4779-902a-5ae8bad72a98]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.494653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ccd14a0c-1757-4fcc-bf18-55f4bbb7ac52]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.521210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49b44630-555e-44ba-9cef-1936996c4258]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.546245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e265df66-fa27-42cc-a70e-787cc1f5d568]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.547080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:00.547620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:00.548071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:00.572562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f42f9d1-7b98-4450-a219-e04852031531]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 413671, 'reachable_time': 25356, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161102, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.573683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:00.574173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:00.574850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-074757da-387c-402b-9869-921d663de888" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:54:00.575173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-074757da-387c-402b-9869-921d663de888" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:54:00.575598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:00.575920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:00.576322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '64b6770e-d1f5-4b4d-97d6-b3e2b9c29645', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:0c:ad:3c', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '074757da-387c-402b-9869-921d663de888', 'device_owner': 'network:router_gateway', 'standard_attr_id': 3069, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.213', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:53:57Z', 'updated_at': '2022-07-01T09:53:59Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-64b6770e-d1), ns(qrouter-074757da-387c-402b-9869-921d663de888) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:54:00.596700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:00.597645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f21e47e2-bd33-4711-9fe2-8b7e31ae6867]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.602403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c7a8312-972f-42f2-8c66-901c89afd776]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.603586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-64b6770e-d1, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:00.604017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:54:00.605836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-64b6770e-d1, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:00.606513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-64b6770e-d1, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:00.606917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-64b6770e-d1, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:00.607335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-64b6770e-d1, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:00.607768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-64b6770e-d1, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '64b6770e-d1f5-4b4d-97d6-b3e2b9c29645', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0c:ad:3c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:00.654461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a5772da-97d7-434d-ab71-ca256dc33d35]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.693078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c414c435-aab4-42db-9081-7415180dfb91]: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:00.693078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:00.693078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:00.693078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:00.693078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:00.693078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:00.693078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:00.693078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:00.693078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:00.693078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:00.693078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:00.693078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:00.693078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:00.693847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c414c435-aab4-42db-9081-7415180dfb91]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.694796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-074757da-387c-402b-9869-921d663de888" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:54:00.695333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-074757da-387c-402b-9869-921d663de888" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:54:00.711237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9610b947-254b-463f-8445-508f945f7c6d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.924806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2faa7db6-7e6a-42ce-999d-f3147e5ed7fa]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:00.950095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d19041a-b3d3-41cd-ba9d-ebdaa564313f]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:01.124586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47031af3-b1cb-49a8-8310-92103e299fe2]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:01.125254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:54:01.125693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-074757da-387c-402b-9869-921d663de888" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:54:01.126110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-074757da-387c-402b-9869-921d663de888" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:54:01.126527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-074757da-387c-402b-9869-921d663de888" released by "process_external" :: held 0.551s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:01.127482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:01.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:01.132393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2db8ac3e-dfb8-48a3-97fa-0925b18df53a]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:01.132979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:01.133273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:01.133524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:01.133858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:01.137930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ed76fbe-9b51-4dfd-9b29-63fcb52db9d6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:01.144502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91ace212-0dab-4270-9d39-5265d48b6197]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:01.146008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[003c4d39-f201-4a9c-b04a-54394aaf39f1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:01.196014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b42af266-983c-4d44-a376-3eb40e8e48cf]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 413671, 'reachable_time': 25356, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161135, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:01.206763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:01.207673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20f9f84a-e7f4-4673-95ff-48105886fb15]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:01.209210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:53:57Z", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:53:59Z", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:54:01.215061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:01.224953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id dae8712b-79d9-46a8-9666-415e4e322611. Wait time elapsed: 2.266 Jul 01 09:54:01.225207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id dae8712b-79d9-46a8-9666-415e4e322611. Wait time elapsed: 2.267 Jul 01 09:54:01.225523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:01.987605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-db166f90-bc07-41ca-ac06-918a1a970696 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['074757da-387c-402b-9869-921d663de888'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:54:05.227741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.002s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:05.228324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:05.228945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:05.229840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:05.230122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:05.231293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a5bb6bc-d8f8-43f9-993a-43feb7dd10fc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.253059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6cb000f8-0979-4794-bfa0-971d8f97f70f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.284428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7babba7-e7d6-4739-a84a-1501f689f606]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.320014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e77af60d-a195-480b-b881-4dea94248f28]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161155, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.342932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f5318c0-dd33-489c-94a6-5abf0a58486f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.375508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8bdc8bf-fd18-445f-a36f-8bdf54e9a2d0]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.409805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29f091dc-a88b-4960-977d-5ab20143fbcf]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.437471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9654dfa8-bb60-4b38-9930-60ef39f914d3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.438355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:05.438854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:05.439857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:05.441765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:05.486119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:05.486539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a063f502-0189-49f0-b520-e264524346e6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.490088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1f6686d-c8b3-446d-815f-b3a14ede36f3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.491946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:05.492356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:54:05.493149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:05.493951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:05.494467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:05.496326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:05.497119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:05.542437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6d15795-4d3e-410a-83e3-ca718bb56913]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.591649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[226d3769-713c-47d8-ac8d-8b143e0ff44a]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:05.591649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:05.591649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:05.591649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:05.591649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:05.591649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:05.591649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:05.591649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:05.591649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:05.591649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:05.591649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:05.591649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:05.591649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:05.593261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[226d3769-713c-47d8-ac8d-8b143e0ff44a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:05.594714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:05.596003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:05.597591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06683a58-4e16-40ca-be18-c0174dabcd78]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.598237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:05.598868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:05.599531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:05.600112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:05.601409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ac8f8db-9d64-4be4-bf9a-5994fb024410]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.611620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71421e7f-74dd-4ccb-bf57-4a0494536cbb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.613324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[553dc474-85fe-4a57-9c37-2c7a4c546d06]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.658071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6803871e-67d4-4354-8385-4ed998dc6d08]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 414158, 'reachable_time': 31116, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161175, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.673969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:05.674741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fee9e00-fd6d-4d76-85cf-87a5fb9c45bd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:05.677660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:03Z", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:05.681716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:05.697521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 1, update_id c59d1a83-8da6-4d68-ad8a-12d1ce5473ea. Wait time elapsed: 3.699 Jul 01 09:54:05.697521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 1, update_id c59d1a83-8da6-4d68-ad8a-12d1ce5473ea. Wait time elapsed: 3.700 Jul 01 09:54:05.697521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:06.761901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.073s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:06.762957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:06.764933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:06.765409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:06.765823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:06.767674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b0311b2-635c-4114-958f-1508930aade0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:06.789430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1ad1620-a1af-40f2-adb3-040d1e697d04]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:06.813034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[170743b6-2e96-45d9-bc77-40e3f580c0b7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:06.844404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6594fe69-77aa-443e-9982-45573a53bc97]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161232, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:06.858559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60425223-669b-424e-bd5b-d58756dd7e6b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:06.886964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1256c97c-e0e0-4c1a-b1de-dbf42756956b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:06.908613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c05623b2-6c09-41d8-8150-2b01c887864d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:06.934134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9250510-6e65-41f6-a561-a6b4701c6a95]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:06.935006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:06.935346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:06.936690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:06.936989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:06.952682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:06.952893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[671c1faf-3942-4797-981b-6fbdb4f39edf]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:06.956720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8d23473-76c6-4e67-bacd-0c6be0bf24c0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:06.957769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:06.987731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:06.988533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:06.989059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:06.989540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:06.990304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:07.046257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d31cadd-6d26-4da9-9de6-ddf2871b956b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:07.084391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[16163775-7818-43a7-bc3a-f196807264b1]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:07.084391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:07.084391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:07.084391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:07.084391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:07.084391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:07.084391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:07.084391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:07.084391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:07.084391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:07.084391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:07.084391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:07.084391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:07.085468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16163775-7818-43a7-bc3a-f196807264b1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:07.086561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:07.088201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:07.091078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c28b35e-f157-4a09-a5b2-d001212e42f1]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:07.091724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:07.092052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:07.092304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:07.092809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:07.093925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42d3deaf-ad73-4bc4-b1e4-adb2b5ee8784]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:07.094941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a5acad6-3233-4ec4-a583-cd4d466ff067]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:07.096038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6764a04f-a3a0-48d5-8c2f-af28e5acd6f7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:07.115963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5475a232-93fe-4702-b06e-8ed605311151]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 414310, 'reachable_time': 35180, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161250, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:07.121913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:07.122366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb03c6ed-e807-4877-9068-4a1842efdef3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:07.123639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:03Z", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:07.126902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:07.139571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id dae8712b-79d9-46a8-9666-415e4e322611. Wait time elapsed: 8.169 Jul 01 09:54:07.139571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id dae8712b-79d9-46a8-9666-415e4e322611. Wait time elapsed: 8.169 Jul 01 09:54:07.139571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:09.335984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.208s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:09.336543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:09.337157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:09.337477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:09.337757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:09.340434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f47794e-247c-4b33-bdab-d93ac42e1333]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.355663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[137d2c53-08ba-454e-8dd2-0c0a14049f3b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.377620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ff838a6-048f-4db0-8d0d-0a3639acabc7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.404247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eeebadb5-62b9-40ff-a01e-4cc3a84b973a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161256, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.424825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f6ab19c-a60b-48da-adf3-5774bda43bce]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.446441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a17043cd-7ee7-480f-8d27-b9d8a04f86e3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.468040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a164b7b1-8e47-4a34-983d-a13eb1c2b28b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.488237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b43fdb5-1626-4df0-a9ac-dab659b566c4]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.488775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:09.489043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:09.489302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:09.489573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:09.503703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:09.505140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06722db4-e074-4ea3-be21-572f62da44e8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.507683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77ddb5ea-f535-4a75-95e4-d2bc752aeff4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.508678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:09.551782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:09.552488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:09.552919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:09.553314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:09.553882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:09.600735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11562640-b5cb-4576-9039-742d801e80de]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.628660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[da6d3c54-433c-4044-9156-60c9658de765]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:09.628660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:09.628660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:09.628660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:09.628660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:09.628660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:09.628660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:09.628660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:09.628660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:09.628660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:09.628660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:09.628660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:09.628660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:09.629542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da6d3c54-433c-4044-9156-60c9658de765]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:09.629785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:09.631075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:09.632795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf0d2e9d-2f7a-4c51-9153-5c4732dd2919]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.632795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:09.633103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:09.633294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:09.633659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:09.634640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c78bc64f-b4f9-40a9-a5af-13fd7cdd2bff]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.636079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bde87933-15a1-4f79-957d-1cb6fea5971e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.636952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbe9b764-22cc-4bf7-a9fc-a68fba5b0edb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.657072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4110a6b6-1b68-4730-93f7-08457b5e8672]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 414566, 'reachable_time': 26368, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161274, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.662272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:09.662684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1345bf3-18d4-4148-a0f7-951261035feb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:09.663841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:07Z", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:09.666316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c59d1a83-8da6-4d68-ad8a-12d1ce5473ea. Wait time elapsed: 7.677 Jul 01 09:54:09.679315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c59d1a83-8da6-4d68-ad8a-12d1ce5473ea. Wait time elapsed: 7.677 Jul 01 09:54:09.679315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:10.364683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.699s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:10.365565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:10.366634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:10.367103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:10.367593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:10.369067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c49c6c5-9625-49e6-bed8-ba13f60a7049]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.391749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4f892ef-93fb-4575-bcf3-0713818ceace]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.414039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20a6b280-2a09-4ee8-96c4-e69b551b2ae8]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.458730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4e151d7-dd5f-4a1e-a121-42b835862a8d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161278, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.482170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40867990-9ebf-46a9-9f42-c21c0ec38fec]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.505723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd58c494-4e91-4281-8270-33aa47f49b70]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.536129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41354cc9-76de-4d4f-a4f1-220bc0087ce8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.564788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3a19dfc-6c56-4318-9612-d89668009c89]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.565363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:10.565770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:10.566180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:10.566496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:10.583408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:10.584022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce537598-aebc-4344-8a3b-b9d58a169c60]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.586681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d6bacf4-4d48-48ec-ae78-7c461bdf34f9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.587814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:10.614235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:10.616193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:10.616861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:10.617504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:10.618422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:10.680377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[330dab87-666d-47a2-b031-775f7df543dc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.746339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a5332b85-367b-44a4-b6df-98493dd0a717]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:10.746339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:10.746339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:10.746339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:10.746339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:10.746339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:10.746339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:10.746339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:10.746339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:10.746339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:10.746339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:10.746339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:10.746339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:10.747853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5332b85-367b-44a4-b6df-98493dd0a717]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:10.749240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:10.752562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:10.756549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16b501b6-f353-4abb-abe0-75cb06cc59d5]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.758499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:10.759182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:10.759658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:10.760253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:10.764054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f2c7cf8-1b5c-4199-ab9f-f1bede931cc8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.765485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45a1f38a-acf3-41cf-9f15-22ddb6668594]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.766668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dadc58b6-e268-4e31-bfc1-a7af7ca1be1c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.810515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a16abf14-c917-4070-9adc-c4d292d9f520]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 414672, 'reachable_time': 33636, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161303, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.818715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:10.818975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38241947-9501-4a60-b2f1-99ac595f7491]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:10.820814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:09Z", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:10.823936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id dae8712b-79d9-46a8-9666-415e4e322611. Wait time elapsed: 11.863 Jul 01 09:54:10.838075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id dae8712b-79d9-46a8-9666-415e4e322611. Wait time elapsed: 11.864 Jul 01 09:54:10.838075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:13.642670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.819s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:13.643255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:13.643953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:13.644491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:13.645244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:13.646532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e79a20e-587e-4e25-85de-3eba62e12f3c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.659790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50d66102-27d7-4112-8f39-8220cb66628e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.680362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15037ad3-b996-4044-bff8-e3524b0f13a3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.714178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecef3f9b-fe62-4cc4-b2df-31af6824dee5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161382, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.734998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25ad5afd-2741-4f71-a43a-d671bbba944d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.755042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46684604-a135-4a03-bb0f-bc5222190b0e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.774885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b355249-9cb3-4992-94f3-dd43bd6aad38]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.795494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[287c96f1-1d80-48c1-97a0-c9fdeed961fa]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.796132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:13.796470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:13.796797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:13.797220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:13.811568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:13.811953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6e9aadd-86e9-4eb7-b470-c17b9952c8c4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.814616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb3d8b6e-f4b4-4d18-a8c9-6cd2524bfe00]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.816030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:13.847318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:13.848483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:13.848875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:13.849259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:13.849638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:13.896579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71f07ce8-ee14-424c-992f-1d58f417fbce]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.945231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1811dd2c-7058-452d-a730-0b0ead2cca1e]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:13.945231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:13.945231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:13.945231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:13.945231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:13.945231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:13.945231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:13.945231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:13.945231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:13.945231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:13.945231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:13.945231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:13.945231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:13.946121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1811dd2c-7058-452d-a730-0b0ead2cca1e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:13.947273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:13.949743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:13.951585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d769bd50-1e6e-4631-882b-73ceee3ecb44]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.952259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:13.953427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:13.953957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:13.954259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:13.955905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2f0bba6-1d1b-4467-81a7-88eb21049215]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.956944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f90de7b-1b89-40ce-8692-44c59aabcaea]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.957825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb9ad81c-c18d-4082-8cc2-678d1014890b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.986987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae15d5c9-9658-4dd3-a2be-062784a7f6b9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 414997, 'reachable_time': 22708, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161400, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1d7cd75-c17c-4132-a6b7-86f62aa45952]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:13.992123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:12Z", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:13.995490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c59d1a83-8da6-4d68-ad8a-12d1ce5473ea. Wait time elapsed: 12.006 Jul 01 09:54:14.008907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c59d1a83-8da6-4d68-ad8a-12d1ce5473ea. Wait time elapsed: 12.006 Jul 01 09:54:14.008907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:14.783568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.789s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:14.784137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:14.784895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:14.785286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:14.785635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:14.786775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[683bac96-2387-4fa3-9c12-7a4a8426d601]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:14.799211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[100406bf-3cbc-4a26-b3f2-30c2c50103c9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:14.819981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0751bb8-77bb-48b1-bccc-0603ee7e3133]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:14.847736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f23b0fa1-fd8d-4fa4-99f1-e13cb31ab243]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161403, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:14.870621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5742f936-4ace-47b3-ab3a-359961791d73]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:14.892520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[739745b5-9d4f-4af1-be86-fb5cdeebc22f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:14.918004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68dc2637-3459-4567-a890-f664ed9caef3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:14.938352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1225a810-97a0-427c-bfc1-53079656b038]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:14.938950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:14.939213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:14.939511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:14.939793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:14.955504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:14.955799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a612f10e-c28e-44c7-9502-822b0dcc3ec1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:14.958455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[834fc9d1-17a9-4c20-956e-2dd1aaac3cf4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:14.959528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:14.994403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:14.995336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:14.995754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:14.996136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:14.996509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:15.044748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d34219db-7375-47cf-b455-d0a9cba60609]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:15.073709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[59c016bd-81e0-424a-b8d1-47c90590015e]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:15.073709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:15.073709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:15.073709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:15.073709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:15.073709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:15.073709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:15.073709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:15.073709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:15.073709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:15.073709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:15.073709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:15.073709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:15.074673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59c016bd-81e0-424a-b8d1-47c90590015e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:15.075029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:15.076654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:15.078993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2f3a57e-a7d8-4ea1-934a-7884164d3150]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:15.078993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:15.078993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:15.078993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:15.079310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:15.080047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2659e92c-3cbb-4624-b3c7-43e0316d5f98]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:15.081009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[edd2173c-ec63-4091-bf5b-f5926ed22e72]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:15.081899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5337c0d7-143b-410a-9863-c34ae153a335]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:15.100655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff040731-ea90-4805-8fd0-680dfe09d891]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 415111, 'reachable_time': 19264, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161422, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:15.106419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:15.106690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.005s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:15.107189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:15.107437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:15.108292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:15.108588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be1b223c-6140-4201-9017-eb744104bc9f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:15.109989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:13Z", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:15.112729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id dae8712b-79d9-46a8-9666-415e4e322611. Wait time elapsed: 16.153 Jul 01 09:54:15.125990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id dae8712b-79d9-46a8-9666-415e4e322611. Wait time elapsed: 16.153 Jul 01 09:54:15.125990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:15.171847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:54:18.223241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.111s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:18.223842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:18.224604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:18.225145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:18.225412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:18.226676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[352e126a-4f49-4e15-ab99-ae228c9aeccc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.243124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae7bd34b-a15a-48cc-8337-1657d537641a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.264519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a4c7210-e9a2-4948-b5bd-acb56686fa4a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.293361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[928dd4cb-8bb1-4f0b-80f5-20a4304fe565]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161475, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.314210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4e3a30d-b420-4a43-86d5-075231140aab]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.335719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d281b4b3-0bb2-4e97-92ea-2769348830e8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.361462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f20b3309-8eee-4ad8-b421-7b528462d64b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.382019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87cd9128-5035-4440-b4b9-f7cd191b91a6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.382613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:18.383136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:18.383766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:18.384126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:18.398921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:18.399308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38ff1e3a-6aaf-462c-9914-b46c4da47789]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.402665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55a60ded-05d3-407e-887a-3f0b9e1fce85]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.403664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:18.438249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:18.438920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:18.439314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:18.439786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:18.440167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:18.482722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ca55b0b-61a4-4ee0-9209-8982f9a9a308]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.527610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cb04df9d-ca91-4ad2-9b6d-46b499eb3dd4]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:18.527610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:18.527610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:18.527610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:18.527610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:18.527610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:18.527610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:18.527610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:18.527610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:18.527610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:18.527610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:18.527610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:18.527610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:18.528398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb04df9d-ca91-4ad2-9b6d-46b499eb3dd4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:18.529283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:18.530578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:18.532213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bef0a705-c3b3-4756-af1f-b5f6614346a2]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.532213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:18.532552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:18.533200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:18.533200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:18.534102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be3beb20-f91b-4cdd-bc3b-d471008314a5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.535063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed566469-d5cb-4232-a976-b03f09674144]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.536050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07887d72-b324-4022-9ba0-24371efc476b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.554383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a071175-3097-4c59-b866-d6ae3f7666fa]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 415455, 'reachable_time': 18796, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161493, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03462042-e115-46e7-88af-9f14bdf72842]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:18.560043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:14Z", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c59d1a83-8da6-4d68-ad8a-12d1ce5473ea. Wait time elapsed: 16.572 Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c59d1a83-8da6-4d68-ad8a-12d1ce5473ea. Wait time elapsed: 16.572 Jul 01 09:54:18.564404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:19.276509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.716s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:19.277133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:19.277689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:19.278012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:19.278374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:19.279555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18136a97-b093-4a4b-aceb-bbff3795ea4c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.295565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e435e40b-a157-43ca-8ac1-59da7dc7742a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.319282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fba920bb-ba94-4058-9239-9b70d922a32e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.374291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f723137-d805-43cf-96ec-b7770434c061]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161496, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.400171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eca62c71-811b-4494-a311-47a8edde0dbd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.423353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aef73493-3177-4a81-a2b6-4032f0fbb657]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.438994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffd13e1c-1b2a-4ed7-993e-5cc567e47af7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.460434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd2fcc15-0289-4ebe-9314-fff8a5e38a18]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.461132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:19.461478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:19.461978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:19.462479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:19.479225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:19.481263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c98dd8f2-0266-42c6-8ea1-b76add5aebf0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.484270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[678cb785-409f-4c67-b7da-51f394c52c76]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.485593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:19.521746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:19.522415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:19.523502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:19.523899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:19.524506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:19.571029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[445373ed-fcf0-4797-bfa4-4a75d3c43e99]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.601853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0f0d2528-baa0-419d-997c-2436827cdfcd]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:19.601853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:19.601853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:19.601853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:19.601853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:19.601853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:19.601853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:19.601853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:19.601853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:19.601853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:19.601853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:19.601853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:19.601853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:19.602829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f0d2528-baa0-419d-997c-2436827cdfcd]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:19.603230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:19.604937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:19.607721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fabd9d3f-94e9-4ad5-bd11-2dfc566bf410]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.608558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:19.608883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:19.609137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:19.609479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:19.610461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49927701-5cf3-497d-b97a-1dd32bb1b8b9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.611436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29d1a18d-798b-4f32-b097-3a2f35783bbd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.612318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ea743c4-714e-4ea6-8b21-f5b02dff04f0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.630813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75714e12-8e40-48c2-b468-ce69be9b7e57]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 415563, 'reachable_time': 35076, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161515, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8486cbdb-dc19-46c2-95b5-ac77340faa5e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:19.635500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:18Z", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:19.638339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id dae8712b-79d9-46a8-9666-415e4e322611. Wait time elapsed: 20.678 Jul 01 09:54:19.647769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id dae8712b-79d9-46a8-9666-415e4e322611. Wait time elapsed: 20.679 Jul 01 09:54:19.647769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:21.998008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-f22dc561-bb3e-42f8-b602-9c89a50bfb0d tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['074757da-387c-402b-9869-921d663de888'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:54:22.295164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.657s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:22.295355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:22.297823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:22.298220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:22.298690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:22.300132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74c811e9-e48d-4201-861d-e448fa5292c7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.326310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44662089-59dd-439f-ad10-a640fdc1fe73]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.351250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c7003e5-bd27-4915-99c8-e4c1912a8402]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.391867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d0db7f8-1ee9-4f0c-b011-546d312009d6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161566, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.415486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[501be2f7-227f-4dbc-83b7-98f3538a1161]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.441944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fa636bc-1169-49b4-86ed-a692da865d73]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.465608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d79ff454-2a43-41ad-b97c-c49281b5bff8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.487694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8809343-5ed4-46ab-85d8-78717168729f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.488713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:22.489187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:22.489459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:22.489761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:22.505374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:22.505653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f538bda8-c3e0-4af4-8b77-1e680bebd7a1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.508661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e48a8dbb-aa04-4aa4-be03-797c28d9fcdd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.510425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:22.544133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:22.545012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:22.545606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:22.546136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:22.546660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:22.594389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4f38e8f-1ccb-491c-9c1a-6cbf144c4546]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.628360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e498ded6-4fa0-4b5d-bc42-17425c610a28]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:22.628360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:22.628360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:22.628360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:22.628360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:22.628360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:22.628360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:22.628360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:22.628360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:22.628360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:22.628360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:22.628360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:22.628360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:22.629188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e498ded6-4fa0-4b5d-bc42-17425c610a28]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:22.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:22.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:22.632778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30ff58ff-1947-40c2-8f5f-e5af46859606]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.632778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:22.632778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:22.632778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:22.633063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:22.633908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f19fbe1-24c9-4169-b246-470179e73d3f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.635695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c48d5e57-0b6b-457a-82d8-a28c721570e8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.636557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55e6b972-d3de-4e43-b44d-990965ac7b6c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.656442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[264eee6d-931b-4487-8b4d-7c38ac591a19]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 415865, 'reachable_time': 42912, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161584, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.658932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:22.659501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64098c99-a97f-42f9-99b9-7daf4aba2b74]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:22.660699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:21Z", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:22.664589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:22.674319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 074757da-387c-402b-9869-921d663de888, action 3 Jul 01 09:54:22.674597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 1, update_id 338848ac-929e-45f4-ad21-b32b24b93919. Wait time elapsed: 0.677 Jul 01 09:54:22.674839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 1, update_id 338848ac-929e-45f4-ad21-b32b24b93919. Wait time elapsed: 0.677 Jul 01 09:54:22.675145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:23.496664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.821s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:23.497239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:23.498457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:23.498894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:23.499183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:23.500341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1296598-e371-49f0-adda-31c9f765930c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.514664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64689614-4c17-41cf-9109-a33a0fceabf2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.537953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d09d6550-390f-4168-a667-41a180fc4a88]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.564624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95a84455-4e95-4b04-ba09-f060d7b2c9c3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161587, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.586904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b15fc729-f949-4a58-ad0b-fd63e565d4e7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.607079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b48b0828-e53c-4d9f-b35c-8e5a78710c04]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.627111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ceee857-536e-4b2c-854a-1e1579077407]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.638553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b784b58-2e78-4fcd-a757-05032f5e2524]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.639232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:23.639522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:23.639786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:23.640065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:23.655023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:23.655312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b87dc8a7-2565-41a9-a202-bbb77a38bec9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.658010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f194a15-5540-4579-b214-c230f7d8cf67]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.659037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:23.698871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:23.699398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:23.699804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:23.700196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:23.700605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:23.743162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4b549ee-ac86-45f0-8764-882d1e7bbc01]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.792924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a307084f-06ca-4d4b-8990-64562bfeffde]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:23.792924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:23.792924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:23.792924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:23.792924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:23.792924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:23.792924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:23.792924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:23.792924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:23.792924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:23.792924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:23.792924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:23.792924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:23.793925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a307084f-06ca-4d4b-8990-64562bfeffde]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:23.794152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:23.795761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:23.797438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f0ed0c8-3469-481c-9a51-96f8596c9d13]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.797438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:23.797610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:23.797754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:23.798245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:23.799251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56f39192-daa8-43a9-8c89-98f7de55e3d9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.800193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5e955f4-29de-4f0a-9251-c4c04b8691dd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.801064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee5b09f6-6a9b-432e-ada9-991c9deb6fca]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.820968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93965970-b71a-4e95-952a-5c9431add06a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 415982, 'reachable_time': 28408, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161606, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72e6f263-2708-43ee-b560-ad1e947055ae]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:23.825541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7cd220ad-8b78-40fb-9feb-856b8399ec64", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.251", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "85023afa-c185-476b-bade-caea382e0b9d", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3099, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:df:53", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "29d6e7e7-0e9d-421e-b7e7-2f6a789b701c", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:21Z", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:21Z", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:23.828680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:21Z", Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c59d1a83-8da6-4d68-ad8a-12d1ce5473ea. Wait time elapsed: 21.839 Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c59d1a83-8da6-4d68-ad8a-12d1ce5473ea. Wait time elapsed: 21.839 Jul 01 09:54:23.840315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:26.468253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.641s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:26.468811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:26.470157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:26.470588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:26.471496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:26.472849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[baf709ee-79b5-4939-8c78-fa58fb5965f0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.488708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83322f9f-92de-4d9c-9372-3daba9bf3416]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.511830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a6e41a0-8471-4bf1-a64f-10a2c065015a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.541435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86b7a9fe-440b-47e5-9fdd-836a045fad0a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161614, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.568070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9180b8c1-3a88-474c-9a8f-21519e9938cd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.590497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0de7aea-6d74-48bc-827d-db41f9f7513e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.610844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f22932a0-7373-4531-b9a8-3e50e81c5a8a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.621520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2189298-acb4-4910-92ff-12ac3c8ba33f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.622094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:26.622364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:26.622648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:26.622920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:26.637831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:26.638013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d6d74e0-d66c-4352-8663-2ce83c8bff75]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.640648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b47706ea-1a81-4485-ad1b-aad8f4b1fd18]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.641580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:26.677473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:26.678151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:26.678549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:26.679091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:26.679509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:26.733492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f694e66-0ce5-4edc-a4ac-ffaf024baa86]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.767415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[03cf80db-030e-45a2-8d49-6b340c915bea]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:26.767415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:26.767415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:26.767415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:26.767415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:26.767415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:26.767415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:26.767415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:26.767415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:26.767415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:26.767415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:26.767415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:26.767415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:26.768384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03cf80db-030e-45a2-8d49-6b340c915bea]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:26.769103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:26.770548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:26.772368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc0861d9-f79a-4b67-a140-a8c518b76c8e]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.772368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:26.772368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:26.772609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:26.773606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:26.774656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6139d762-fc47-4472-8cdc-d1cc88294117]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.775997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c316fe88-17f7-4b80-bf1e-180043006af0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.776906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b32697c-5d30-44ed-9ef5-ffbc0bd69631]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.795647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19447393-4df5-45a7-9e94-5eb4e8baa744]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 416280, 'reachable_time': 36508, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161632, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.801597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:26.801851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf608f00-93b6-484c-bea5-e1989a4caecc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:26.802884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7cd220ad-8b78-40fb-9feb-856b8399ec64", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.251", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "85023afa-c185-476b-bade-caea382e0b9d", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3099, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:df:53", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "29d6e7e7-0e9d-421e-b7e7-2f6a789b701c", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:21Z", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:21Z", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:26.805654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:23Z", Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id 338848ac-929e-45f4-ad21-b32b24b93919. Wait time elapsed: 4.807 Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id 338848ac-929e-45f4-ad21-b32b24b93919. Wait time elapsed: 4.807 Jul 01 09:54:26.816932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:27.686624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.881s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:27.687264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:27.687890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:27.688277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:27.688562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:27.689740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba01122f-4432-4d47-95f3-8f37b5798072]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.706293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c36bf905-c581-4a0f-9cf9-e43648553ffb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.726594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[712bc615-3ba6-4c3f-8049-4f5af39ec922]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.754009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[221d90ed-dec1-437d-b34c-b1789dbc758a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161680, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.774232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1a631af-3a2c-42d3-b321-33c90cb7a975]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.794835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6255b359-a9bd-4914-9600-20cb27e2a061]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.815547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a636771-edb5-4583-a9d6-05c3bb061c36]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.835688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c495e434-bb5b-4838-a09f-f2ab5735f6a5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.836234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:27.836512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:27.836778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:27.837058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:27.852118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:27.852415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45555559-9c51-45e6-89ae-6391541f79a1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.855772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1d94d2e-1bd2-484c-bbe5-1dec16f35fa0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.856735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:27.890545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:27.891387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:27.891820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:27.892205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:27.892586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:27.935730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f6067b8-1d47-4ae9-bee3-1ab2161a7725]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.966547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b313929a-1a90-4628-ace2-0e4a07a71d59]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:27.966547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:27.966547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:27.966547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:27.966547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:27.966547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:27.966547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:27.966547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:27.966547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:27.966547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:27.966547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:27.966547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:27.966547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:27.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b313929a-1a90-4628-ace2-0e4a07a71d59]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:27.968740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:27.970279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:27.973216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a900440-45d3-45e3-8198-6e98eea6c32b]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.973216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:27.973216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:27.973216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:27.973216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:27.974473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9067c438-9319-4598-90f8-b2a3ccd04e58]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.975560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b0d4bfa-4940-4be1-8ff9-ef71316a0bcc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.976548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9023852b-8abc-4d9f-974d-c2c2efd39f61]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.996447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e46a6ef1-c756-4631-b36b-6ba440af4dfb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 416401, 'reachable_time': 16520, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161698, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:27.999119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:27.999647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dcc8333a-1bfd-4df9-8981-d13d4904e95b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:28.002140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7cd220ad-8b78-40fb-9feb-856b8399ec64", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.251", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "85023afa-c185-476b-bade-caea382e0b9d", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3099, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:df:53", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "29d6e7e7-0e9d-421e-b7e7-2f6a789b701c", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:21Z", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:21Z", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:28.005891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:26Z", Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c59d1a83-8da6-4d68-ad8a-12d1ce5473ea. Wait time elapsed: 26.018 Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c59d1a83-8da6-4d68-ad8a-12d1ce5473ea. Wait time elapsed: 26.019 Jul 01 09:54:28.017018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:30.602112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.595s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:30.602726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:30.604080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:30.604521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:30.604737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:30.605896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ceda7e50-c328-411b-b1b9-1032405a9282]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.618536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bc47284-a8b3-4269-89ca-0b4df07e4198]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.639447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7917cb63-4a8b-4c23-813b-15188501f53e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.666655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a63c4184-5782-4b3f-9654-a110b1f8e1b7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161705, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.687795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6f54e89-514d-4b5f-b91c-f8b0db5446c7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.710009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95128299-9c81-4c3a-b7e2-64a16adb3690]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.734095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e17664c-0e05-4610-8d0c-50444e4d524e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.755343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[466e78ce-d118-4741-b3e4-d103314db5ec]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.755935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:30.756237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:30.756507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:30.756787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:30.775901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:30.776884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5cca8d5-6038-4c08-82b6-6459fecd3857]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.779990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51b64cc4-3fe6-4db9-a70e-51e1446f3530]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.780988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:30.817622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:30.818297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:30.819314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:30.819894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:30.820436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:30.865926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf684655-3157-4204-944a-21088e8223ee]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.901044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[58835306-2c1c-427c-90c5-0b1a82c8bb76]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:30.901044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:30.901044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:30.901044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:30.901044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:30.901044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:30.901044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:30.901044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:30.901044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:30.901044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:30.901044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:30.901044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:30.901044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:30.902009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58835306-2c1c-427c-90c5-0b1a82c8bb76]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:30.902416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:30.903980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:30.905752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9919bef7-3b73-46f5-91d0-7195cd28a4a0]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.905752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:30.905752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:30.905752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:30.906647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:30.907667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e398977b-679c-46d7-b1c8-808e8b23b6e6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.908632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e83a60dc-6bf0-408b-9fc7-d8604c43d2a4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.909526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51a96f8e-e91d-44dc-be50-064e34a9e331]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.928801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a73bb69f-2a75-473c-8fc2-e31fb1b0d3da]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 416692, 'reachable_time': 40500, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161723, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bf616f5-def5-4010-80f3-eb5d3362ad90]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:30.933950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7cd220ad-8b78-40fb-9feb-856b8399ec64", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.251", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "85023afa-c185-476b-bade-caea382e0b9d", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3099, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:df:53", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "29d6e7e7-0e9d-421e-b7e7-2f6a789b701c", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:21Z", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:21Z", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:30.936946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:29Z", Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 25, Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 074757da-387c-402b-9869-921d663de888, action 3 Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id 338848ac-929e-45f4-ad21-b32b24b93919. Wait time elapsed: 8.937 Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id 338848ac-929e-45f4-ad21-b32b24b93919. Wait time elapsed: 8.937 Jul 01 09:54:30.952052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:31.765540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.830s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:31.766164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:31.766759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:31.767082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:31.767397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:31.768851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[987e9e49-e4fc-426b-8451-e4f9e8f8fc1e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:31.780706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49c59bb5-6cc0-4db8-b553-370e804df129]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:31.797320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[089d6899-df39-4205-9163-4079f73ccb5d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:31.835541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[566f4098-ede6-4ffe-b358-f134b7bb3e28]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161726, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:31.856940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6826073-36bb-4e92-8ab9-3190e5006356]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:31.878812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[974653f7-8aae-4d72-a10d-38c56a3c8dd6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:31.906148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a55acb71-5fda-4c97-8edf-30ee3f96617b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:31.929481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab5a2edd-0f65-424e-813a-5952329ee9b0]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:31.930415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:31.930415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:31.930611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:31.931064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:31.957584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:31.958992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c24263d4-bb02-4694-9e27-72d941cd226c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:31.966670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1dd007dc-84db-4f8b-a46f-f3d940c2ee09]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:31.967914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:32.007997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:32.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:32.011997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:32.013835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:32.014404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:32.076740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc33d0c9-1f5a-4e52-8663-e9cedd6c4b4f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:32.124520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[92708e08-0ede-4229-9ed8-4deb466088b6]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:32.124520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:32.124520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:32.124520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:32.124520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:32.124520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:32.124520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:32.124520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:32.124520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:32.124520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:32.124520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:32.124520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:32.124520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:32.125314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92708e08-0ede-4229-9ed8-4deb466088b6]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:32.126334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:32.128045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:32.129952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0dd3e582-4ad5-4e48-9f1a-873358848653]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:32.129952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:32.130133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:32.130251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:32.131774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:32.133042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa054e44-656c-4f96-a626-edb4abde5947]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:32.134158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b3555ef-db55-4085-aa95-d4bce9316144]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:32.136088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76fa6d27-fe0a-4b09-b612-b76172cd7a55]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:32.193824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63593ca6-6bcb-49f8-b17b-88ca08fc44c1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 416810, 'reachable_time': 28000, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161766, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:32.208243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:32.209884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07f93f44-0eb9-4e85-a061-998bce0bd945]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:32.211243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7cd220ad-8b78-40fb-9feb-856b8399ec64", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.251", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "85023afa-c185-476b-bade-caea382e0b9d", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3099, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:df:53", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "29d6e7e7-0e9d-421e-b7e7-2f6a789b701c", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:21Z", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:21Z", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:32.216305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:31Z", Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 27, Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:32.238992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:32.245581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id 338848ac-929e-45f4-ad21-b32b24b93919. Wait time elapsed: 10.238 Jul 01 09:54:32.245581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id 338848ac-929e-45f4-ad21-b32b24b93919. Wait time elapsed: 10.245 Jul 01 09:54:32.245581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:34.763155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.520s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:34.764143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:34.768000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:34.768330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:34.768617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:34.769863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[438e85ef-53fd-407b-a714-f3da1dafa3b3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:34.784762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f15641b-1898-4c05-af85-957793c43de8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:34.806312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbe03684-cb7a-4505-ba16-3e5271124703]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:34.839840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8d0f3fb-8754-495b-b75b-3028c6fe4208]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161799, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:34.861316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e7b1319-4d5b-4c90-9d5c-1f06a6e21ef9]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:34.882228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3518628c-5562-42cc-8c31-59dfb9dc2e79]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:34.903057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[127f9dc0-b97c-456f-814e-44f17a5ff503]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:34.923551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0efc3b2-1535-48a9-9dff-4ded84d71f66]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:34.924157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:34.924434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:34.924722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:34.924996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:34.939010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:34.939299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b06d5d0-6686-41e7-9d5d-6c7ec6365ff2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:34.941842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33b3d485-b668-4869-a15d-286b2e2c9a2a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:34.942805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:34.977868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:34.979221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:34.988778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:34.989225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:34.989610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:35.038965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33ae9c40-1d9c-46e2-9269-95979ab57d3a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.073613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1b3c527c-ab3f-4390-bc52-23dbbeb55eda]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:35.073613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:35.073613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:35.073613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:35.073613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:35.073613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:35.073613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:35.073613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:35.073613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:35.073613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:35.073613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:35.073613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:35.073613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:35.074742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b3c527c-ab3f-4390-bc52-23dbbeb55eda]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:35.075755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:35.077203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:35.079218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47273c19-eeb7-4c16-b974-2068783c49a5]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.079218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:35.079443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:35.080209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:35.080209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:35.081525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f441ec44-de58-493b-856e-e0d3cf98484f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.082735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[feeccdab-e9bd-4a8d-89a9-c8485aa6794d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.083824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fad85c0-f3f8-4ff3-8e04-6b1937daf8c5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.104482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[543d53f5-f3e9-4908-9770-9faba636be17]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 417110, 'reachable_time': 40668, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161817, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.106708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:35.107104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03c76cd7-fc8a-41bb-95f8-fc60adef68cc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:35.109702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7cd220ad-8b78-40fb-9feb-856b8399ec64", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.251", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "85023afa-c185-476b-bade-caea382e0b9d", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3099, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:df:53", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "29d6e7e7-0e9d-421e-b7e7-2f6a789b701c", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:21Z", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:21Z", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:35.112629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:33Z", Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id 338848ac-929e-45f4-ad21-b32b24b93919. Wait time elapsed: 13.116 Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id 338848ac-929e-45f4-ad21-b32b24b93919. Wait time elapsed: 13.116 Jul 01 09:54:35.123944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:35.844437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.729s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:35.845183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:35.846156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:35.846543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:35.846881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:35.848084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53b53898-4b2b-41eb-b151-05524f50b8c4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.863956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87d4c4e4-81e6-4b0a-9928-fdcf6698ccdf]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.875340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f65a340c-c33d-4aa0-a30a-0bd38f2ab7c7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.905529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f03330a9-991b-430f-b5f9-c7edfa8ab604]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161822, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.925507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[734b399c-4d41-49e9-87be-6759215e85da]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.945704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ee97577-ed67-437e-8ba7-005d49bfd9a4]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.966775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e23931f6-d5ed-425c-96f0-a3bf22fce8d4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.987558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91fa442a-d370-4d9c-8428-8173e0662bf8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:35.988509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:35.988790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:35.989061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:35.989342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:36.005042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:36.005042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd8560ca-6a4b-478c-9618-949a65441acc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:36.009926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfa198d6-fbb9-42d9-9e37-00baaccaa7c7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:36.011263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:36.054398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:36.055072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:36.056707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:36.057454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:36.057847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:36.100053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b269fb6d-60c1-4ea2-a880-17fe15f27db7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:36.155873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4cd0c68c-cbeb-4cab-99b7-464e12f00f36]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:36.155873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:36.155873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:36.155873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:36.155873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:36.155873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:36.155873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:36.155873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:36.155873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:36.155873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:36.155873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:36.155873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:36.155873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:36.157475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cd0c68c-cbeb-4cab-99b7-464e12f00f36]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:36.158754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:36.161376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:36.165937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3513e26f-2db8-4d33-a119-8ed023d663c8]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:36.167085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:36.167439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:36.167735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:36.168078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:36.169147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[828a2e44-d079-4386-a029-1a0ab98d2afd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:36.170629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f69c0a7-ec0d-4338-affc-3b3a0186086b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:36.171671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88879496-6df7-4696-8765-063d12f0a2c9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:36.192835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df6bc650-5ff4-4950-a789-0d2c0963eef8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 417216, 'reachable_time': 42108, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161841, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:36.199447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:36.199447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07268902-4017-4451-91b2-89690dcb8a03]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:36.200396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7cd220ad-8b78-40fb-9feb-856b8399ec64", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.251", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "85023afa-c185-476b-bade-caea382e0b9d", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3099, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:df:53", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "29d6e7e7-0e9d-421e-b7e7-2f6a789b701c", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:21Z", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:21Z", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:36.204526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:34Z", Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 31, Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id 338848ac-929e-45f4-ad21-b32b24b93919. Wait time elapsed: 14.207 Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id 338848ac-929e-45f4-ad21-b32b24b93919. Wait time elapsed: 14.208 Jul 01 09:54:36.213946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:54:39.176432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.970s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:54:39.177059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:54:39.177846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:54:39.178801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:39.182403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:39.184711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b67ab60-6c26-46b9-8d21-888308787773]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.202283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f1dc32b-e7bc-45ca-a099-0c6cffb66a95]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.226588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23010c08-ca96-41f7-8f53-ae833398fd48]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.256927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6937547a-d326-48af-9feb-7dfd985675a3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 161893, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.281505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34678577-c84c-46ba-adc9-76b684ee1c58]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.305739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fc52c5e-ad60-4bd8-9702-9661c8f63a6e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.326870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de88c63d-c763-4352-9bb2-7f226c9b9b37]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.354121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91768bce-df68-4a91-a6d0-5312f07659bd]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.354647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:54:39.354918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:54:39.355207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:54:39.355515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:54:39.371132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:54:39.371476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04a10d3d-518c-4849-824b-dd67818619d4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.374070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d1257e0-8533-4f4d-8774-63a13b8013e0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.375005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:39.404747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:39.405421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:39.405818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:39.406193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:39.406740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:54:39.478066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ab812fd-c0d2-4e02-a765-4032e8193ef7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.512941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7d097bca-f3f4-46db-9a28-60a250a397a2]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:54:39.512941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:54:39.512941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:54:39.512941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:54:39.512941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:54:39.512941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:54:39.512941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:54:39.512941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:54:39.512941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:54:39.512941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:54:39.512941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:54:39.512941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:54:39.512941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:39.513797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d097bca-f3f4-46db-9a28-60a250a397a2]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:39.515578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:39.517254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:39.519499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03296f1c-072e-40d3-87f3-89a770950598]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.519499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:39.519499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:39.519775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:54:39.520309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:54:39.521117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d031cdf-64c2-4229-917b-d562ae33d1d7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.522144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c2d40e2-29e1-4830-9127-475a5a3609e3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.523110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b1d2ea8-8c08-4a11-a1cd-94540bd52b41]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.544774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dde45bc9-ba5a-497d-9456-8527f0068e7f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 417552, 'reachable_time': 32756, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 161911, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.551088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:54:39.551401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75779151-4598-441c-90bd-61a3c7f8b54f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:54:39.552451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7cd220ad-8b78-40fb-9feb-856b8399ec64", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.251", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "85023afa-c185-476b-bade-caea382e0b9d", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3099, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:df:53", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "29d6e7e7-0e9d-421e-b7e7-2f6a789b701c", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:21Z", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:21Z", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:54:39.555162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:36Z", Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 32, Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:54:39.567332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 074757da-387c-402b-9869-921d663de888, action 3 Jul 01 09:54:55.173278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:55:15.103505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:55:15.104158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:55:15.104158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:55:15.104305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:55:35.174668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:56:15.102980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:56:15.103711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:56:15.103711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:56:15.104066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:56:15.176250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:56:55.178242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:57:15.103106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:57:15.103812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:57:15.103812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:57:15.103986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:57:35.179746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:58:15.103727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:58:15.104401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:58:15.104401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:58:15.104553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:58:15.180697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:58:55.182268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:59:15.104215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:15.104935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:15.104935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:15.105113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:23.213693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8c60cbfe-9143-4d4f-93aa-e27eda64cf0c tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['074757da-387c-402b-9869-921d663de888'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:59:23.215802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 1, update_id c8e5f7b1-7e69-416d-9906-b58ae1eb2733. Wait time elapsed: 0.001 Jul 01 09:59:23.216270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 1, update_id c8e5f7b1-7e69-416d-9906-b58ae1eb2733. Wait time elapsed: 0.001 Jul 01 09:59:23.217678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:24.103265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.887s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:24.103820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:24.104859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:24.105249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:24.105605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:24.107066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a9bee4f-f915-461d-87c0-491c983d46a6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.123844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4de39a7-d206-4dc4-8961-ec113f6dce94]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.149258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3ee236c-f10b-440e-a77f-64aedffd6b9b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.194109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7874c70-1810-4f21-9f63-0d1cd4a36e87]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 166813, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.228789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97b8b35e-cd50-43b0-b980-299803f7cf85]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.250269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3627a13-71f7-4bb4-8a33-05e52d1ec077]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.272769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f531ab1-ac1f-4400-9bdd-1cf58e47468a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.294668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[600071f1-eaa9-4e36-84bc-d65382ce8b5d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.295247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:24.295574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:24.295852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:24.296325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:24.312149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:24.312432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02364aa3-3a3e-49ad-a594-7a5ce472f523]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.315787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91faa929-9af6-41b8-ae76-3306dff4c23d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.316811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:24.357317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:24.358857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:24.360813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:24.362037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:24.362978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:24.433352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b4c0181-7d3e-4875-bb18-8f304d0474fc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.475772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cb4dbeb1-c2b4-495c-84e6-08272d71e8cb]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:24.475772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:24.475772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:24.475772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:24.475772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:24.475772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:24.475772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:24.475772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:24.475772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:24.475772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:24.475772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:24.475772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:24.475772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:24.477491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb4dbeb1-c2b4-495c-84e6-08272d71e8cb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:24.481323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:24.483814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:24.489632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc46f34a-8d42-41d5-8352-5467406b11d1]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.490665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:24.491012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:24.491946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:24.491946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:24.493153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1c39374-3495-407c-9124-e331b4aa380c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.495952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[248e8141-8a47-4ae4-977c-62cd70b13f15]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.497159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e80a57d-ee20-4435-aa87-8ccc284de73d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.524558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08843076-7a0b-4919-a04b-4dd7ccb2001d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 446045, 'reachable_time': 20860, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 166832, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.533301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:24.533838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ed14bb9-15a4-44be-a21e-8918acb06a99]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:24.535162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:41Z", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 37, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:24.539273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:24.551329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c8e5f7b1-7e69-416d-9906-b58ae1eb2733. Wait time elapsed: 1.335 Jul 01 09:59:24.551329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c8e5f7b1-7e69-416d-9906-b58ae1eb2733. Wait time elapsed: 1.335 Jul 01 09:59:24.551329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:27.064131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.513s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:27.064700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:27.065360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:27.065681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:27.065959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:27.067069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc6d92a5-eeb0-41a1-a633-b2ee6679f06e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.083141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[324fa823-83be-414b-9e33-49f41edd9819]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.103988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2ad429b-e72b-480e-aaea-c1984db3c1c4]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.131705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d347678-5d13-48dc-8e09-2cdc48864ab8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 166883, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.151642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e2fef17-751d-4f84-b7e4-3fb91234fe15]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.172070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa1fb1c6-ce46-45c8-afc8-484a9d323d1c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.193174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91e3a894-a663-414d-b06e-db78d8d9bf24]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.213330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ac229b0-9749-4fdd-9a73-6a47ed2040b7]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.213866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:27.214140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:27.214505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:27.214786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:27.230003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:27.230325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1723894a-30b2-4294-9fb3-d9a2ba3ff089]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.232941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28f07421-4e0d-4c0a-a1ed-52b524a71f44]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.233919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:27.269865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:27.270792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:27.271272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:27.271902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:27.272902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:27.328630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3586ecdb-61fb-4b33-8b66-6aadd45a5849]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.366083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[91c8ee73-3c6d-40e4-abf7-87d2d98f379b]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:27.366083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:27.366083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:27.366083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:27.366083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:27.366083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:27.366083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:27.366083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:27.366083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:27.366083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:27.366083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:27.366083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:27.366083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:27.367186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91c8ee73-3c6d-40e4-abf7-87d2d98f379b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:27.367985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:27.369870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:27.372339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64bd511a-bee0-4c44-b26e-4d9a07e5c31f]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.372339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:27.372557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:27.372889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:27.373196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:27.374441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d885ef6-dd2d-4cd6-86e0-0e4b102d0d4f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.375702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b55b831f-6738-4050-852d-cb3f08d24f1b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.376729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[732a3ca5-734a-4df9-b98a-73d7b7ea3823]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.400191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d49249e-b6b1-4404-a452-85940d21f428]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 446339, 'reachable_time': 23472, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 166901, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.406720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:27.407579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ba8d52a-8656-4b22-9003-3a8e87397d8e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:27.408742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:41Z", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 37, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:27.411924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c8e5f7b1-7e69-416d-9906-b58ae1eb2733. Wait time elapsed: 4.196 Jul 01 09:59:27.429574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c8e5f7b1-7e69-416d-9906-b58ae1eb2733. Wait time elapsed: 4.196 Jul 01 09:59:27.429574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:28.219272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.808s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:28.221034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:28.222276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:28.222975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:28.223487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:28.225753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2883caa7-48ec-4036-9aaa-728f256d6d69]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.242969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7088adff-bbf7-4223-8996-b3a32340fb2a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.265682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45a3fc50-55ec-4dcd-aec7-26b27d59b00c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.320954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[904b066e-d3af-474c-9efd-4cdb61f00308]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 166907, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.342658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[853ab8e0-5874-46da-b1b1-a92bdb7546b6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.366122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca7f4329-0fd0-40ba-b4fe-ad77616cbb88]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.388791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6015925b-ac21-4d76-9201-383158d05119]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.411446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9583cc7-9666-4004-a722-e0c1bb85dd4e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.412120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:28.412511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:28.412869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:28.413241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e4a89f32-790c-4054-9005-dccf546997cc) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:28.430587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:28.431178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d9250eb-769a-4fa6-aec9-5159d363648a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.440101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bb6a33e-c323-4b1c-8d59-92613eafc3c2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.441894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e4a89f32-79, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:28.475174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e4a89f32-79, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:28.476546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:28.478011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:28.479420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e4a89f32-79, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:28.480322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e4a89f32-79, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e4a89f32-790c-4054-9005-dccf546997cc', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:79:3f:85'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:28.516868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f03e580-b583-4781-9dc4-e9be66338001]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.564485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[47f41fac-d18c-48a8-ab78-ad00d24d15a5]: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:28.564485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:28.564485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:28.564485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:28.564485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:28.564485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:28.564485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:28.564485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:28.564485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:28.564485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:28.564485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:28.564485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:28.564485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:28.565547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47f41fac-d18c-48a8-ab78-ad00d24d15a5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:28.565679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:28.567583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:28.569858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15fed942-6640-4b6a-9964-4c691df0ea8d]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.569858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:28.570063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:28.570282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:28.570613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:28.571684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4fb9dc7-d08a-47ac-8501-6f65098080ee]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.572703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81ba8653-e228-4c00-a19d-623e92d58a2d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.574271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ade60a9-3f96-4b2a-89c5-76cf3a37715a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.595976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aaf45d68-d027-46f4-ad86-b39a4b9f7708]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 446458, 'reachable_time': 17840, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 166925, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.602474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:28.602867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fc16ca6-ffb0-412e-a3b2-8bd25eb230d8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e4a89f32-79 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:28.604061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:01Z", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4a89f32-790c-4054-9005-dccf546997cc", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "0fe3c0d2-4579-4662-8752-1dadeffaed17", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:79:3f:85", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3075, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:54:00Z", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:26Z", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 38, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0f22236c-12d3-40f1-bcd6-daccb6f4a842", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [ Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "1.2.3.4" Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:28.607654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:28.619786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c8e5f7b1-7e69-416d-9906-b58ae1eb2733. Wait time elapsed: 5.395 Jul 01 09:59:28.619786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 2, update_id c8e5f7b1-7e69-416d-9906-b58ae1eb2733. Wait time elapsed: 5.399 Jul 01 09:59:28.619786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:29.213040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-64565d63-bae5-4c15-9ea4-0539ccce01d1 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['074757da-387c-402b-9869-921d663de888'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:59:30.772115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-08c11754-848e-41ba-a0fa-e5f9cd9be939 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got router deleted notification for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 09:59:30.833860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.218s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:30.835412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 074757da-387c-402b-9869-921d663de888 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:30.842834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (074757da-387c-402b-9869-921d663de888), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:30.843524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:30.844442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:30.847279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e4122ed-ac23-4a40-97de-729265623995]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:30.890053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe71f9a4-80f8-4885-b873-753fd987c407]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:30.972854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[180f65a5-a688-4afe-a619-a3e4ddcc9a60]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.009595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5645f35-c437-46d4-a7d8-4ae7822eac15]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 166947, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.033586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7dffede-6677-4726-b2e3-db3d610b0c9c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.061049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f522887-9d5f-4356-8404-cf83b6720859]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.127978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec763510-c650-4372-878f-da05d7ad1277]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.155875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[283bad8f-e285-4eda-a666-e9bace4ba817]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.157218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:31.157714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:31.158068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 074757da-387c-402b-9869-921d663de888 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:31.184915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5aa49da-a500-4b35-a7a8-d634c6beeb3a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 446727, 'reachable_time': 36532, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 166968, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.185727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:31.186005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:31.186434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-074757da-387c-402b-9869-921d663de888" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:59:31.186698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-074757da-387c-402b-9869-921d663de888" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 09:59:31.187026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:31.187283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:31.188219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '64b6770e-d1f5-4b4d-97d6-b3e2b9c29645', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:0c:ad:3c', 'admin_state_up': True, 'status': 'ACTIVE', 'device_id': '074757da-387c-402b-9869-921d663de888', 'device_owner': 'network:router_gateway', 'standard_attr_id': 3069, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.213', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:53:57Z', 'updated_at': '2022-07-01T09:54:02Z', 'revision_number': 4, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-64b6770e-d1), ns(qrouter-074757da-387c-402b-9869-921d663de888) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 09:59:31.208581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:31.208839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48b2a094-3336-4e80-94df-4ecf87e2dbad]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.212042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[580d74b9-01e0-4841-afdd-705d8a2ccf63]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.213290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-64b6770e-d1, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:31.263792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-64b6770e-d1, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:31.264911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-64b6770e-d1, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:31.265587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-64b6770e-d1, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:31.266797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-64b6770e-d1, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:31.267600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-64b6770e-d1, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '64b6770e-d1f5-4b4d-97d6-b3e2b9c29645', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0c:ad:3c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:31.323996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb9cdd01-8138-482f-b376-f055dd26c0d1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.356394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[370a6b9a-7008-4c8c-9979-3d752a29f0e1]: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:31.356394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:31.356394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:31.356394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:31.356394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:31.356394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:31.356394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:31.356394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:31.356394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:31.356394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:31.356394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:31.356394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:31.356394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:31.357428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[370a6b9a-7008-4c8c-9979-3d752a29f0e1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.357428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-074757da-387c-402b-9869-921d663de888" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 09:59:31.357726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-074757da-387c-402b-9869-921d663de888" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 09:59:31.383988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25350e13-04fd-4c19-80d6-33c2dac18da3]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.590435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d20970d-dbe1-42dc-ae7a-72e6e8e0b339]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.603182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[236c1f4a-0f5c-431a-8ae6-0a67b2405a0c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.790510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45b239e2-20c7-4a7d-aa78-b427bd540cfe]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.791189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 09:59:31.791566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-074757da-387c-402b-9869-921d663de888" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:59:31.792000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-074757da-387c-402b-9869-921d663de888" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 09:59:31.792480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-074757da-387c-402b-9869-921d663de888" released by "process_external" :: held 0.606s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:31.793353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:31.797005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:31.801193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10ffcd20-336b-4e9f-9613-36ae37ad1019]: (4, ['qrouter-074757da-387c-402b-9869-921d663de888', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.801998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:31.802330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:31.802589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 074757da-387c-402b-9869-921d663de888 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:31.802935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/074757da-387c-402b-9869-921d663de888.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:31.804166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d79dd02f-4e39-4ec1-bd56-65fb64be1369]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.805178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af62436d-f5a4-49c3-b773-b912ffe6e136]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.806201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e66cdde-49de-445a-b77f-b8bc586aa125]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.830117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a5807f5-1955-4085-8f2a-f788aa668854]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 446727, 'reachable_time': 36532, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 166985, 'error': None, 'target': 'qrouter-074757da-387c-402b-9869-921d663de888', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.842668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-074757da-387c-402b-9869-921d663de888 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:31.844888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45be5ade-3075-4ccc-8398-4e65a17c8766]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 074757da-387c-402b-9869-921d663de888: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-64b6770e-d1 not found in namespace qrouter-074757da-387c-402b-9869-921d663de888. Jul 01 09:59:31.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-498195474", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213" Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:56Z", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:28Z", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "64b6770e-d1f5-4b4d-97d6-b3e2b9c29645", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0c:ad:3c", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "074757da-387c-402b-9869-921d663de888", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3069, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.213", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:53:57Z", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:54:02Z", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 074757da-387c-402b-9869-921d663de888, action 3, priority 1, update_id 86f150c2-e6b8-4d50-aa6a-c26b6a115398. Wait time elapsed: 2.633 Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 074757da-387c-402b-9869-921d663de888, action 3, priority 1, update_id 86f150c2-e6b8-4d50-aa6a-c26b6a115398. Wait time elapsed: 2.634 Jul 01 09:59:31.849362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:31.892662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.045s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:31.892992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 074757da-387c-402b-9869-921d663de888 was not found. Performing router cleanup Jul 01 09:59:31.896936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c53338d9-d151-4072-abd7-917a219b4249]: (4, ['qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:31.897641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:31.897957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:31.898276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 074757da-387c-402b-9869-921d663de888, update_id 86f150c2-e6b8-4d50-aa6a-c26b6a115398. Time elapsed: 0.052 Jul 01 09:59:33.671300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-a23212e2-b757-4e84-9662-306a8de966d3 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['4ae39661-aad4-4b8a-8f84-f7daeabb6e80'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:59:33.672395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id 4de65400-8b05-4dd6-8e17-b9fd01c8ea23. Wait time elapsed: 0.001 Jul 01 09:59:33.672730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id 4de65400-8b05-4dd6-8e17-b9fd01c8ea23. Wait time elapsed: 0.001 Jul 01 09:59:33.673118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:35.184764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 09:59:36.323101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-4cd5e3a0-588b-41b8-af4e-66500bc1d4f2 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['4ae39661-aad4-4b8a-8f84-f7daeabb6e80'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:59:36.907188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.234s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:36.907991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:36.909039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:36.909458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:36.909847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:36.911439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[633b4031-0b4d-4ba3-92a4-aa7013c18d47]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:36.928898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a794ecb3-a60a-4843-9605-c32e3589ae2f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:36.957818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3dbfb1af-3044-4e36-b149-b5ee441b61b4]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:36.992227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b59cd68-cd48-4a78-9e3e-0747ce380e05]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167022, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.029733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0500f5d1-0def-40b6-b4ff-241edb95fd1c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.053802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a85c90ba-3b1b-4b97-9c21-f0282222f1c4]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.080607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14026969-716f-4142-b25c-9a2a3687f320]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.119792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4bdcbcc1-6364-4455-8a39-3adf2d711039]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.120624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:37.121197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:37.121692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:37.122316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:37.146627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:37.147530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3eb4252-3fa2-48b1-a8ae-e7e20a30566c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.151775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bff47d8e-39ed-48ae-80ab-8f3658ec922a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.152981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:37.153487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 09:59:37.154241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:37.154909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:37.155303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:37.155714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:37.156095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:37.201530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53d3b78c-34cb-4198-b2bb-c67be427f42c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.259593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0e0fcb47-7e1d-493e-9211-c38c266b192a]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:37.259593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:37.259593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:37.259593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:37.259593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:37.259593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:37.259593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:37.259593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:37.259593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:37.259593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:37.259593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:37.259593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:37.259593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:37.261075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e0fcb47-7e1d-493e-9211-c38c266b192a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:37.262241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:37.265102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:37.270420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a956a2f3-cc1d-47f5-b7f5-0e24f3df9b15]: (4, ['qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.271197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:37.271558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:37.272217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:37.272568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:37.275070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99c0ac5b-8435-422d-93b5-a52e2da8574c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.276431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1adeaa4c-77b8-4b75-9dba-dcdd38fd3181]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.279336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14f4527b-09da-4195-89e2-6b3dfd5a1da6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.318772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c35b8404-def3-44fc-8ec4-2c5ebf5f7704]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 447325, 'reachable_time': 44256, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167050, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.323652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:37.323652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f410af26-d9e4-4365-b9e4-2c72f6259080]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:37.324582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id f177fa9f-5601-4b6d-813e-58c73a580b35. Wait time elapsed: 1.002 Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id f177fa9f-5601-4b6d-813e-58c73a580b35. Wait time elapsed: 1.002 Jul 01 09:59:37.327925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:39.011261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.685s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:39.011969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:39.012769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:39.012894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:39.013085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:39.014449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0733219-1b2f-43fc-bb12-0e9955107c31]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.029253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77f02c9d-1b81-49cb-8863-3947c19fde9d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.056760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44e6f4ce-cb6f-4429-9a76-7e10ba761e3c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.086228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8442b29-bce0-4d9b-9c2b-2132fffc9852]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167097, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.119505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cff9015-1c05-400c-a1cd-09ea83624b1a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.141721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb20608b-b35e-4807-b892-d0b3aaec9add]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.167240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38d38c46-cab0-4c61-8d55-f0a35eb21215]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.198334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8ba590b-1f9f-4fcc-8e5d-945f79a49435]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.199090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:39.199450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:39.199762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:39.200131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:39.224437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:39.224615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1dfba9c9-247c-453b-9e04-39025b7788c8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.228013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7661ad04-d0e0-4c91-9e63-c97f7a04b655]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.229135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:39.274154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:39.275768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:39.276318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:39.276886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:39.277418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:39.323896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30d4eb37-7b3c-4322-9c1e-5e2ea62ba7a2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.359547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[04e4aae8-51ec-42ae-839b-7383f46f8213]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:39.359547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:39.359547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:39.359547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:39.359547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:39.359547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:39.359547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:39.359547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:39.359547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:39.359547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:39.359547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:39.359547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:39.359547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:39.360573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04e4aae8-51ec-42ae-839b-7383f46f8213]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:39.360868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:39.363711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:39.371209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4a759ee-3c40-4b55-958d-1d238462a90e]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.375281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:39.375700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:39.376182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:39.376963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:39.378184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19c763d1-1bb8-41f1-83a7-0a92afb2703c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.379522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[859c71b4-15f5-4011-82e9-6ccd77583708]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.381212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[110d0b48-26f2-4314-bc42-ae5e54a10623]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.401106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b979078d-683d-46ec-a86a-cd6db1b3a381]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 447534, 'reachable_time': 23656, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167116, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cdaf0b48-4169-4646-ac40-600cf560ffca]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:39.405471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:38Z", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:39.412181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 4de65400-8b05-4dd6-8e17-b9fd01c8ea23. Wait time elapsed: 5.736 Jul 01 09:59:39.431265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 4de65400-8b05-4dd6-8e17-b9fd01c8ea23. Wait time elapsed: 5.736 Jul 01 09:59:39.431265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:42.120327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.712s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:42.120914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:42.121545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:42.121877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:42.122162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:42.123406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3b950fb-d48f-45ef-a3be-d5cf280ae060]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.139025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[318f4efe-d67c-43b9-8871-21d5f4ca038f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.161890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6257c724-88d6-4fd5-8849-dda4fd8a2630]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.197050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fcd5f7f-64cd-460a-9066-cc53176bb5fc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167168, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.219527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d97f4fa6-0b73-49f7-8dc7-18d729cb831c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.242906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c95fb134-c7d0-4bf4-b7e7-cccf0422af49]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.263210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4112fc2b-bcf8-4071-b68b-c9dd8a4dd10f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.284674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad8cef40-d65a-45a0-a15d-80d85eed2e5f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.285302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:42.285832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:42.286163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:42.286507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:42.302301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:42.302657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d139c59-bf71-4524-9c47-5ba97a8dc03f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.308797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d458505-98ca-4876-b7eb-58469eb90cd3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.309740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:42.336689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:42.337570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:42.337963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:42.338348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:42.339462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:42.382838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[631ce933-efcd-4a1d-ba6d-9173fdcf4410]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.414805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9b0d48e4-97df-4dbc-8aff-f3e675aa1aa1]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:42.414805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:42.414805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:42.414805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:42.414805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:42.414805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:42.414805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:42.414805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:42.414805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:42.414805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:42.414805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:42.414805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:42.414805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:42.416248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b0d48e4-97df-4dbc-8aff-f3e675aa1aa1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:42.417683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:42.419243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:42.423118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a952281-1daa-45e2-aed8-32981ad6e312]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.423959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:42.424430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:42.424912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:42.425263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:42.426322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c1c83db-551c-41e7-a63c-15090648f94b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.427587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7523d68-e794-4a99-bdee-2475c0d5562a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.428667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75eb3641-b8fb-4496-8284-7bd1573bd5cc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.448382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87dddb46-7cd3-4463-b037-c17fd259a89e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 447846, 'reachable_time': 24912, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167186, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.450681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:42.451102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[865189aa-66b6-4135-a5c3-622c1deddf27]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:42.452254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:38Z", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:42.459612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:42.472364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id f177fa9f-5601-4b6d-813e-58c73a580b35. Wait time elapsed: 6.137 Jul 01 09:59:42.472364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id f177fa9f-5601-4b6d-813e-58c73a580b35. Wait time elapsed: 6.137 Jul 01 09:59:42.472364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:43.162779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.702s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:43.163475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:43.164641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:43.164959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:43.165302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:43.168292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2373972-a0e6-400f-a279-2418daa7b663]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.182572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5461f362-6b65-49b9-9558-34cc126c52e3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.206477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2417d5e-d5db-45b8-968e-99ec32b216af]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.232715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[195fb93b-2d58-423e-99b0-3c3086ab78f3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167189, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.253878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6033d2b-831c-4112-9d57-f001cfc32c65]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.279400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b45bc689-0890-4cf8-814c-655016d96890]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.300523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8964179d-48e9-459e-88aa-92da1eb29fea]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.320789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67307552-885d-4675-b80a-0e2a4aa41b59]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.321351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:43.321633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:43.321901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:43.322179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:43.336567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:43.336854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd088159-ae5e-4cfc-9166-411fb8b1a868]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.339383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea9caa9f-14a3-4a7d-8d21-7172ec21ddc9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.340307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:43.381833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:43.382771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:43.383163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:43.384793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:43.385213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:43.431192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ed8c378-aea0-4c61-b74b-d00f7c58b21b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.462922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d666bff2-90d1-4689-b869-fd5d52cbfc05]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:43.462922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:43.462922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:43.462922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:43.462922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:43.462922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:43.462922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:43.462922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:43.462922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:43.462922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:43.462922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:43.462922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:43.462922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:43.464011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d666bff2-90d1-4689-b869-fd5d52cbfc05]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:43.464106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:43.465752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:43.468080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4976a854-18d8-46ef-9cb4-baaee61f008d]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.468080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:43.468080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:43.468080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:43.468080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:43.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1ca3858-9c5c-4202-9c8d-a0e38084bf60]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.470733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b455b7fa-1126-46fe-b949-4957c2c620de]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.471692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c97a5b2c-b593-414b-b242-890fbb3965d2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.492435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7dde89d-2a89-4f88-b68c-2798c7413247]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 447949, 'reachable_time': 25068, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167207, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.494450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:43.494682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[828724ce-9b41-4eaa-a7f8-0cd846ca75c8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:43.496096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:41Z", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:43.498758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 4de65400-8b05-4dd6-8e17-b9fd01c8ea23. Wait time elapsed: 9.825 Jul 01 09:59:43.512933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 4de65400-8b05-4dd6-8e17-b9fd01c8ea23. Wait time elapsed: 9.826 Jul 01 09:59:43.512933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:46.476741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.979s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:46.477316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:46.478089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:46.478410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:46.478689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:46.479970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27ae32ea-3ff5-4614-ab9e-07452c87c666]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.492957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34ad7c74-9fe1-42f1-8d81-28e7f3a642f6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.514933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ab896ed-4b86-4663-88e3-7cdb97472a07]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.542436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78e58bb2-423b-4aa4-aaf0-bd9632d41fd9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167261, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.565105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67e298d1-5dc2-4daa-96a6-f71318c5b99b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.585300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30baed00-cf76-43e8-a220-ecf76e845eb8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.605963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18b7e845-8841-4327-9582-116953bc33f3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.626640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52e4bb62-6541-41f6-a30f-1bc420d08f7d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.627221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:46.627573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:46.628766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:46.629058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:46.644568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:46.644974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24e6be1d-3153-427b-b06d-8d3677c4981f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.647654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc0d7e8c-98ab-4517-9631-a786e288aaff]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.648584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:46.677375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:46.678330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:46.678801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:46.679428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:46.679880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:46.731811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d6d635a-3331-4a71-ac8a-eb5adbde1283]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.762380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8d2f6f47-90b2-4f2a-9019-02ca9513f591]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:46.762380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:46.762380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:46.762380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:46.762380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:46.762380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:46.762380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:46.762380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:46.762380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:46.762380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:46.762380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:46.762380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:46.762380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d2f6f47-90b2-4f2a-9019-02ca9513f591]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:46.763665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:46.765628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:46.769979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b98a90e7-82aa-4f5d-a9a9-da874f243a98]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.769979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:46.769979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:46.769979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:46.769979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:46.769979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a16af62-be0d-4333-b92f-d95e5bb186c1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.769979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e044deb2-a6af-4183-b135-b27a5b8cc779]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.770898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fb2ec74-2a28-4780-99f9-5d45859130c8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.789473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b658e4a8-c548-42b4-8599-ae1a618b384f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 448280, 'reachable_time': 19412, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167279, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.796095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:46.796095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07030ddb-01a0-49d6-b3e6-419f751e472f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:46.796326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:44Z", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:46.799991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id f177fa9f-5601-4b6d-813e-58c73a580b35. Wait time elapsed: 10.476 Jul 01 09:59:46.812083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id f177fa9f-5601-4b6d-813e-58c73a580b35. Wait time elapsed: 10.476 Jul 01 09:59:46.812083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:47.585465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.786s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:47.586604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:47.587257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:47.587664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:47.587947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:47.589282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[790a3955-5c23-4a02-be55-332d8f1310fd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.607657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fda19e16-1644-4e83-a8d1-587882dab5b8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.630619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17315085-71bd-44f5-8658-256f6cf61007]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.661284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8650a51f-8f1b-49fb-a5c3-0a240955e121]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167282, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.681955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0079ce5a-6615-432e-b9a9-d1e2876c402f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.702479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3bc54b6-1c86-475f-92e5-654d7d7f85fd]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.722574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14da7018-74a0-4bd9-8c4f-e16d09df7fe3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.742921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0eccf32-5d76-4799-95e0-e0c9c87948e3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.743423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:47.743706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:47.743972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:47.744250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:47.764251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:47.764541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d408fea6-42a6-4169-8e27-28219114c703]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.767965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[232a7789-e867-43f9-b7fe-3b2fc6bc5f43]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.769152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:47.805716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:47.806703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:47.807215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:47.807991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:47.810515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:47.885835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13191c9a-c5c2-415c-95fa-8dae1be160c9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.917894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9f86386a-ee35-432b-ab19-f39969b7a34c]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:47.917894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:47.917894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:47.917894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:47.917894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:47.917894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:47.917894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:47.917894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:47.917894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:47.917894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:47.917894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:47.917894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:47.917894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f86386a-ee35-432b-ab19-f39969b7a34c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:47.919505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:47.921391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:47.924602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c982b00-6470-42ef-ace1-b9eca8bf4110]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.924602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:47.924602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:47.924602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:47.924602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:47.925467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e111f626-ec40-4551-8c83-00c33bd2618c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.926442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1206e0bf-6bb8-44db-843c-dcc7601304fb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.927493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c83b0dca-369d-4da3-bf2a-e899188f12c6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.946970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f3c0c79-adc7-44fe-a635-20796a72fda2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 448392, 'reachable_time': 44916, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167301, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.949303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:47.949555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a106277-6fa8-4e84-aa55-a5ba670bc77d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:47.950549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:46Z", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:47.954770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 4de65400-8b05-4dd6-8e17-b9fd01c8ea23. Wait time elapsed: 14.280 Jul 01 09:59:47.969606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 4de65400-8b05-4dd6-8e17-b9fd01c8ea23. Wait time elapsed: 14.280 Jul 01 09:59:47.969606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:50.491081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.538s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:50.491704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:50.492966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:50.493294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:50.493580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:50.495180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31d13483-c478-4c3d-b5c8-50f37c10d117]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.511591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88a9435b-818d-43f4-970d-e7dc9dc00b7c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.533210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a3a6554-8453-427d-9966-0fe7ca0356a7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.564715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91ddeb42-af02-4288-9040-78096a36dbbb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167308, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.585006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d057a410-ad9e-40a8-a257-169b95d786b0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.606387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23730a77-0e1f-4ab8-b890-ab6d66bf5d39]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.629126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12f91f08-f357-42ee-9776-41440db6d3c3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.650340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5adf0397-6d23-4b92-8d4e-6054b60b68a6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.651107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:50.651467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:50.651930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:50.652574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:50.672821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:50.673151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15a261f1-79c1-4e88-b7df-f2e85e736ed5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.676013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cea75a3f-4157-4b23-a5c5-a313f66d65f5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.676957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:50.711041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:50.724616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:50.726286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:50.728478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:50.729100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:50.808127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7eef081-215d-4e57-9c86-f6f7d88ad57a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.866727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[62aa97fd-84e1-450a-a8b0-b6352638fc2d]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:50.866727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:50.866727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:50.866727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:50.866727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:50.866727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:50.866727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:50.866727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:50.866727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:50.866727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:50.866727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:50.866727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:50.866727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:50.868395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62aa97fd-84e1-450a-a8b0-b6352638fc2d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:50.869642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:50.872607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:50.877315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b513bc60-ef01-4e98-9c89-2b116d86ea1e]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.878355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:50.878683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:50.879077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:50.879540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:50.881029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa4dd4ee-dcbe-4973-bf9f-0b94afdd3ff9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.881660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[171f8d8c-0ee7-4f93-bf0b-06630de0902b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.883016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee5e9c1d-355f-440c-ba3d-d2836e3672ef]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.905173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c076490-de18-4df6-8836-bab2916f88d0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 448682, 'reachable_time': 40496, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167328, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.910718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:50.911016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0aca872e-5590-46d7-9792-8f9867d3514f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:50.912223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:48Z", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:50.916941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:50.931194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id f177fa9f-5601-4b6d-813e-58c73a580b35. Wait time elapsed: 14.601 Jul 01 09:59:50.931194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id f177fa9f-5601-4b6d-813e-58c73a580b35. Wait time elapsed: 14.601 Jul 01 09:59:50.931194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:51.646676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.722s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:51.647265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:51.648168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:51.648582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:51.649083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:51.650340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51f50924-08eb-4907-9fc7-0734a8fc0b23]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:51.667698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f68ee8f2-eeb7-4781-82b2-065269302ba6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:51.704636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[272c724f-b3bd-469f-9b79-313a1a213b68]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:51.739894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1058123-d5e8-4769-831b-1fe6d7e08a33]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167340, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:51.774123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2da2106c-a698-452f-b37b-f79f8c76f8ca]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:51.918787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39bc3a2d-b222-4571-9ad8-3767d7c8317a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:51.956844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f54aa53-81d8-42ed-8f77-2d99c8751bca]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:52.018872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[273da88c-bf0d-4c2f-a9ff-2783dab35c06]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:52.019714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:52.020255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:52.021185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:52.021770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:52.049205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:52.049475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[557aa8e7-1ac4-4ce9-9282-54e93d5b608d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:52.053531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45508c64-1042-43d0-9d93-a9a3b81b305d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:52.054550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:52.084557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:52.086546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:52.087120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:52.087843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:52.088395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:52.140962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f75db0d4-fd39-4e66-a287-fd3d557ff63a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:52.178378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[db6cea10-e8b2-4794-a689-ced33c167281]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:52.178378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:52.178378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:52.178378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:52.178378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:52.178378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:52.178378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:52.178378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:52.178378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:52.178378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:52.178378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:52.178378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:52.178378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:52.179440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db6cea10-e8b2-4794-a689-ced33c167281]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:52.180025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:52.181986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:52.184336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1d49c18-4e99-4fa4-bbbe-9c5e002fc58c]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:52.185071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:52.185450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:52.185707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:52.186104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:52.187281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a3640d9-3d0b-4ba4-8854-4490992fec59]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:52.188724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5d7ae4a-d8e5-4eab-840b-5ae2d4197097]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:52.189728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e9be39c-60a7-4ad1-a02d-2dfbe48b8729]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:52.215770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecf67698-d5fa-4f9b-a315-0ae6ad45add6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 448800, 'reachable_time': 22124, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167395, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:52.225465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:52.225728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9bc589c-e959-4211-bdde-c5075752206b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:52.226771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:49Z", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:52.229786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 4de65400-8b05-4dd6-8e17-b9fd01c8ea23. Wait time elapsed: 18.556 Jul 01 09:59:52.240989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 4de65400-8b05-4dd6-8e17-b9fd01c8ea23. Wait time elapsed: 18.556 Jul 01 09:59:52.240989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:54.654228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.426s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:54.654824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:54.655463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:54.656129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:54.656414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:54.657763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a47bdda-f924-465e-adfd-7921de49561e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.674083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92bc9ad7-c07b-4932-bb7b-6fec72230285]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.705584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[569e7ed1-5640-465a-957c-36ba7e75688b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.743889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8223a418-c026-47d9-bdc2-6aaa72f4e071]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167402, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.766065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd03851d-2a0a-4aeb-b5b1-0230d046e4d9]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.788306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[829b5b8e-dbec-4c52-998b-80b618977bd1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.808981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a3d763d-abbd-4b20-8163-7c834240cabe]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.830820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5eea006d-229f-4c5d-ac64-386afb3923d6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.831620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:54.831958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:54.832286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:54.832630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:54.849779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:54.850132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7868dacb-0240-47fb-8775-0dea74df69f9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.852986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3445744-f083-4ce6-b716-141b1fb75bc5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.854797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:54.893351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:54.894928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:54.895552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:54.896090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:54.896624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:54.946291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0eb28ca-722c-451b-b045-7ec7a811f413]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.991953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[24822339-fa04-42f3-b2c4-f759cfe60d28]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:54.991953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:54.991953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:54.991953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:54.991953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:54.991953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:54.991953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:54.991953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:54.991953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:54.991953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:54.991953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:54.991953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:54.991953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24822339-fa04-42f3-b2c4-f759cfe60d28]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:54.993089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:54.994932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:54.997193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e123a46-5fa4-434f-9ef2-59ed8a8be256]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.997193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:54.997193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:54.997193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:54.997193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:54.998011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c3591b3-7b55-4976-b348-344ff233b885]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:54.998958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5decc0b5-69fa-42cb-b235-e613b12698c2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:55.000702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[530882a6-5235-48d8-8f7e-1ab93186e132]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:55.023041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd34528f-fe57-4d67-9154-c5c3e805e210]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 449100, 'reachable_time': 26144, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167420, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:55.030498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:55.030975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e811a026-dc9a-494b-94b5-e633281b1de1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:55.032985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:51Z", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:55.035662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id f177fa9f-5601-4b6d-813e-58c73a580b35. Wait time elapsed: 18.711 Jul 01 09:59:55.053975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id f177fa9f-5601-4b6d-813e-58c73a580b35. Wait time elapsed: 18.712 Jul 01 09:59:55.053975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:55.564530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b5bb7c8e-249d-4899-8c48-c294b33f5559 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['4ae39661-aad4-4b8a-8f84-f7daeabb6e80'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 09:59:55.954935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.919s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:55.955585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:55.956469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:55.957358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:55.958641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:55.960096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33bca158-d617-4f45-a802-c842384a53ee]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:55.979444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3672de6d-4c3c-4ff2-b990-58403f4f1ebc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.010809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f519d45-db07-498a-9528-ab3cfab06280]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.054215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[112c38cf-7285-4048-9058-46947a047af3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167425, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.082095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5fe6b98-0153-4536-af3b-c9916b034dbd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.107875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e71129e-88bd-4a59-9e4d-c5971ec62dfe]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.146068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2810644-34a4-4a8b-aefb-9d97bd8b2275]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.170791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6f9f6d4-c638-41d6-ab9a-3c53ea1c74e3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.174502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:56.174923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:56.176209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:56.176631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:56.210721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:56.211806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3397421f-b03e-4b17-846a-7f0f6e679f77]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.215010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12f6d122-9c88-47b2-a536-c3a6501eb159]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.216662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:56.242675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:56.244289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:56.244827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:56.245353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:56.245974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:56.319082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0021e4e4-31da-4ff5-b28c-210620738ddc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.358512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5508577a-7f5d-4b70-a8d6-0c0210c7a451]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:56.358512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:56.358512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:56.358512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:56.358512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:56.358512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:56.358512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:56.358512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:56.358512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:56.358512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:56.358512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:56.358512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:56.358512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:56.360558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5508577a-7f5d-4b70-a8d6-0c0210c7a451]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:56.361608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:56.364438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:56.368750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1ff3d5a-334b-4f13-9c03-008d2933bfff]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.370586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:56.371527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:56.371856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:56.372310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:56.373600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d3a1512-aaf9-45b7-8d5c-e8dba0b0f4d3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.374798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46f9f980-c64f-4fc2-8c4c-c0e34d0e6aad]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.378765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4298d3ea-182d-4874-a5f4-163d56f9a014]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.415679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0866ab84-282d-4af7-ba3c-cac5e6743cdc]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 449231, 'reachable_time': 33628, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167444, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.432338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:56.432760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1110b1ba-5722-451a-8994-5efb5abbfe53]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:56.433899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:56.437274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:55Z", Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id 07924206-2ebd-40a2-a99b-548053bfc3a3. Wait time elapsed: 0.872 Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id 07924206-2ebd-40a2-a99b-548053bfc3a3. Wait time elapsed: 0.873 Jul 01 09:59:56.455339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 09:59:59.486983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.048s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 09:59:59.488130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 09:59:59.490038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 09:59:59.490675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:59.491206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:59.493180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04b82cb5-52bd-4562-a468-95c25dbbd52d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.516573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a2b207a-339c-4494-bc9d-31c96db62b1d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.538237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[982685e6-034f-4120-baf0-27871b1aa340]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.570273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ca00568-ba81-4058-a25c-89f00eacd41a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167501, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.592388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[109438b1-4f0e-4420-8849-3c40b82cc1e4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.614672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f40f45d6-eea8-42a3-be0b-31b9a5e10fbe]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.637687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b814bdec-6036-4ddd-914a-7384ef405ab9]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.660998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5f853f5-7756-40d7-a09c-7c9fc81cfa2a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.662801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 09:59:59.663169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 09:59:59.663578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 09:59:59.663960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 09:59:59.680781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 09:59:59.681336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6dea8794-f15a-419e-9a22-344f79b159f8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.684065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c063fa9-9523-420f-a2fb-f179f68e9212]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.685143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:59.726322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:59.727355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:59.728144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:59.729013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:59.730100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 09:59:59.792464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3454674e-38b4-454b-8f73-d59b18bd4e10]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.840278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e471c5c0-5104-457a-bf2c-cbb360a95b2c]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 09:59:59.840278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 09:59:59.840278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 09:59:59.840278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 09:59:59.840278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 09:59:59.840278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 09:59:59.840278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 09:59:59.840278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 09:59:59.840278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 09:59:59.840278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 09:59:59.840278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 09:59:59.840278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 09:59:59.840278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:59.841412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e471c5c0-5104-457a-bf2c-cbb360a95b2c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:59.842529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:59.845328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:59.849244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d57d711-785f-4ffd-88d5-01ed5904b77b]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.850002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:59.850483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:59.850898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 09:59:59.851239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 09:59:59.852297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7db9118f-ffb3-40a6-8c25-fe15fbdb8713]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.853321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98c060ae-9913-478d-ba02-a29b60610b63]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.854269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0bf8dd63-ad83-418d-ab3e-8de073117a87]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.875439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e85e6788-2a99-4462-b89b-469baf4235df]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 449583, 'reachable_time': 26368, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167519, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.884057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 09:59:59.884365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b955dfdc-fb39-45b2-bd96-08937d6ab242]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 09:59:59.885419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 09:59:59.888128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:56Z", Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 4de65400-8b05-4dd6-8e17-b9fd01c8ea23. Wait time elapsed: 26.215 Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 4de65400-8b05-4dd6-8e17-b9fd01c8ea23. Wait time elapsed: 26.215 Jul 01 09:59:59.903461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:00:00.727952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.841s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:00:00.728568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:00:00.729528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:00:00.730049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:00.730330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:00.731455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e79ea28-76bf-4448-9699-b5065b97a2c7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:00.755261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1294917-86fa-406b-a46d-e7b0b2268070]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:00.778908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b017a823-fdbd-495b-b8dd-39aa3acacf75]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:00.814431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d7bead9-de95-42e7-9767-2aa83a90c07f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167522, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:00.836377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f766959-9966-4064-af85-134abde0367e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:00.860624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56fdc8c8-a87f-43d9-af4b-20e25ceca7e8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:00.887939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0002339-796e-44af-9c4f-ccd6e478529e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:00.910263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68d78c50-b56e-49cf-aa8e-595adf65e7de]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:00.910978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:00.911473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:00.911913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:00:00.912452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:00:00.930290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:00:00.931036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71b9e199-077e-45f0-9768-1fb4ebec68c8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:00.934386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e2634ea-4af0-4170-86ed-877b7b2336a5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:00.935527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:00.970193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:00.971866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:00.972508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:00.974595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:00.976233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:01.021747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4a839ae-325f-49df-8039-a77dd0d87ffb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:01.055517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d0ace4b6-cc5b-4b19-9a38-67f6eae32f71]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:00:01.055517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:00:01.055517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:00:01.055517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:00:01.055517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:00:01.055517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:00:01.055517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:00:01.055517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:00:01.055517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:00:01.055517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:00:01.055517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:00:01.055517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:00:01.055517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:01.056913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0ace4b6-cc5b-4b19-9a38-67f6eae32f71]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:01.057845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:01.059799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:01.063814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3999367-4ce9-468f-b79a-823b9ce5afc6]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:01.063814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:01.063814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:01.063814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:00:01.063814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:01.064374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25178d0e-c36e-4247-9e8c-ea12f587ac5b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:01.065437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[890fdbaa-feeb-4f78-955a-fae804c8220c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:01.066623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d982dbf3-c865-4c8a-9655-ffc3e3587c1e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:01.093478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2ab55d1-c9bb-40dc-b633-1f19f67b6eab]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 449707, 'reachable_time': 23552, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167541, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:01.102523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:00:01.102859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac4cdee4-7e22-4686-9420-ef28af26520d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:01.103968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:01.112771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:59Z", Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3 Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id f177fa9f-5601-4b6d-813e-58c73a580b35. Wait time elapsed: 24.789 Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id f177fa9f-5601-4b6d-813e-58c73a580b35. Wait time elapsed: 24.791 Jul 01 10:00:01.127223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:00:04.017499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.902s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:00:04.018130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:00:04.018759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:00:04.019155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:04.019622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:04.020742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e77660b-bd00-4a06-a55d-d4f0335f4c75]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.045059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b4ba267-e16b-4788-a1fd-04df441680af]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.067583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d45999c7-0bb9-487e-a4ca-3f84d95025c4]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.095999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56ec8e7b-57b4-499a-b2e4-4daf7297fd14]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167593, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.118252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91dccb9d-23e2-47c8-8c11-b5d2e21a7e48]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.139817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb02a35c-64be-4740-abcc-9b537acfd069]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.160655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[712e98e9-cdef-44a1-b025-37cc9f47c440]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.180143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35bdf926-32d7-4350-8537-6ab588c46986]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.180690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:04.180969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:04.181239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:00:04.181524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:00:04.199170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:00:04.199723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd06d420-e56c-4044-8a75-18e334e429b1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.202982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62f43089-a653-498e-a32c-ace74bc913b0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.204383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:04.249742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:04.250409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:04.250797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:04.251178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:04.255081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:04.296820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0ae2331-0f02-4ac6-883f-570e5dbb0345]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.328918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c4358a35-4c2f-4502-be8e-20d844db397e]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:00:04.328918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:00:04.328918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:00:04.328918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:00:04.328918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:00:04.328918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:00:04.328918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:00:04.328918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:00:04.328918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:00:04.328918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:00:04.328918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:00:04.328918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:00:04.328918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:04.329867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4358a35-4c2f-4502-be8e-20d844db397e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:04.330166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:04.332876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:04.335289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4778190-f772-4f19-8ba5-dcb59d6778ae]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.335289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:04.335538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:04.335771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:00:04.336107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:04.337122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9975a3ba-b060-4576-afa4-2ef7f4465c0d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.338024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7102428-9967-48e4-8265-6d63989523a2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.339015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[691f7d68-e74f-4203-a3c3-4ee81507f85c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.356184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4b71ecd-a137-4421-99f9-13e680bc304b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 450035, 'reachable_time': 17348, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167611, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.362512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:00:04.362763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28a0c61a-bd1b-4e1f-80b6-8b3c82f103aa]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:04.363821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:04.367693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:01Z", Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3 Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 07924206-2ebd-40a2-a99b-548053bfc3a3. Wait time elapsed: 8.800 Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 07924206-2ebd-40a2-a99b-548053bfc3a3. Wait time elapsed: 8.800 Jul 01 10:00:04.379541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:00:05.229479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.864s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:00:05.230110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:00:05.230841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:00:05.231167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:05.231481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:05.232898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[259946c7-026d-4782-a55e-6cd4b628da75]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.247688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04ab9a85-4ce1-4da8-bcde-e8764496f32c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.269731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f71ef0c-96fc-446b-a301-429d36735245]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.297389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b60e8df-88fb-4b02-b56e-87520c490e5c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167614, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.318312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ccddc213-dedb-4e62-9e9a-ce72b2358feb]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.330280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96fdde5a-ab98-450c-9e1a-6c85f022b8b0]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.351177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92e517e8-ecd8-4471-a744-10ff681698ce]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.370930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95544112-eb81-4579-9227-533d55eae60d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.371516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:05.371795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:05.372062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:00:05.372341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:00:05.387291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:00:05.387703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7146567a-f009-4aa7-a83a-0a0d85178d3d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.391100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b68270ff-7bcf-461e-aec9-25f283e728be]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.392083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:05.425158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:05.425834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:05.426941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:05.427660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:05.427772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:05.467808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95760d38-c9e8-428c-9961-b1b9815241cc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.503312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[99483bc8-109d-4345-acdd-9b33943deba4]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:00:05.503312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:00:05.503312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:00:05.503312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:00:05.503312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:00:05.503312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:00:05.503312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:00:05.503312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:00:05.503312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:00:05.503312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:00:05.503312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:00:05.503312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:00:05.503312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99483bc8-109d-4345-acdd-9b33943deba4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:05.504541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:05.506602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:05.509778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42805a48-e95c-4f20-bf0e-e86b602cdd54]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.509778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:05.509778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:05.509778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:00:05.509778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:05.510965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cfc37eca-9f8e-4898-8e93-9ce6ca340acb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.511944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[def6323e-93b3-4ec5-a212-de6157b59fce]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.513005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26a2b673-1e69-403e-ba53-6a61aa4e3690]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.533606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9b1ddff-63dc-47b5-81c9-bf902c61186f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 450156, 'reachable_time': 32492, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167633, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9362ff36-9d68-4468-b79d-697966b3177f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:05.539575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:05.543350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:04Z", Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 07924206-2ebd-40a2-a99b-548053bfc3a3. Wait time elapsed: 9.976 Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 07924206-2ebd-40a2-a99b-548053bfc3a3. Wait time elapsed: 9.976 Jul 01 10:00:05.556467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:00:08.932672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.391s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:00:08.933725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:00:08.934740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:00:08.935171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:08.935602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:08.937668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38d0cdef-8652-409a-aba5-1a87521f600f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:08.955982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7944fadd-53e8-4e79-9cef-cf5d8e8d9aa2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:08.977484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08086eb7-e1a3-440d-a3f0-f38374721c10]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.013112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7e463d6-9059-4319-92ab-b95dfc076d18]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167686, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.036191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fa89f23-95af-4c5e-9f30-8f17184f23d6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.060360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d571885a-1d85-4323-96dc-8f71d483f86b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.082296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[996cb4b7-a160-45e1-8f76-e00cacf8e85b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.094715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a14c090a-b82f-4e85-ae37-b7ddb1b46855]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.095129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:09.095572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:09.095868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:00:09.096215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:00:09.111267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:00:09.111702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c025e6f-9921-4903-905b-7fbb03935441]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.114432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e39da198-7fb6-4e36-8623-f55e11ca1bb5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.115470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:09.145899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:09.146865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:09.147690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:09.148152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:09.149788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:09.180470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4dec9b40-0598-496d-a166-4f4776a665c2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.224610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1afab343-9a31-4d3e-8297-2e3fc6bd9851]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:00:09.224610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:00:09.224610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:00:09.224610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:00:09.224610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:00:09.224610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:00:09.224610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:00:09.224610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:00:09.224610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:00:09.224610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:00:09.224610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:00:09.224610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:00:09.224610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:09.225712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1afab343-9a31-4d3e-8297-2e3fc6bd9851]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:09.225857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:09.228567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:09.232225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8dd9d0de-4a2a-4f5a-9f3c-8a034bbb0f13]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.232976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:09.233292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:09.233852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:00:09.234021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:09.235018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ffa3dd9-0bdb-4be4-9677-c95fe264f9e7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.236079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[918c46b2-0fc3-4e33-8abd-2e2aab239929]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.237144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93e9c84c-f0f0-424a-850f-c2d75a1d3713]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.257430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2616d078-a965-41c8-8999-d09aec492e26]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 450527, 'reachable_time': 44268, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167704, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb5445e5-74bc-4214-9fd0-0435b5f08deb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:09.263173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:09.267653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:05Z", Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 24, Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 07924206-2ebd-40a2-a99b-548053bfc3a3. Wait time elapsed: 13.699 Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 07924206-2ebd-40a2-a99b-548053bfc3a3. Wait time elapsed: 13.700 Jul 01 10:00:09.283112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:00:09.999798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.733s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:00:10.000483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:00:10.001401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:00:10.001835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:10.002120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:10.003387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cbbab0e-14d8-459e-975e-e58a21885a91]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.021069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9810c41c-5270-4de1-900e-cd67cba3b194]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.052901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efbc3894-d5ac-4c9a-b855-19478c0afab1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.083945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[917babf6-912a-45e2-aff9-b9dbba67775f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167707, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.114292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fae9fd06-a4c8-4e4b-8c67-2facc29c1ed9]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.138827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[edc430c8-3b56-4068-8d8a-90343dd2c8e1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.174885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[642c1003-8011-42dd-b37f-184a5a634d19]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.203101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25c2f969-25ab-43da-8282-31437480c367]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.203974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:10.204229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:10.204541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:00:10.204821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:00:10.220464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:00:10.220717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48897af4-13b6-46e8-bfa0-b1c1b33cb516]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.224978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5ce19c4-33a3-4d40-b871-e7f332a6d315]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.225928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:10.268055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:10.269504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:10.269869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:10.270261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:10.270645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:10.302833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3b383af-9ab4-4e8c-bd1c-3db117f47b16]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.338561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ed0a1ba4-1a1f-4008-a978-2c830fb7e5ff]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:00:10.338561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:00:10.338561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:00:10.338561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:00:10.338561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:00:10.338561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:00:10.338561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:00:10.338561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:00:10.338561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:00:10.338561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:00:10.338561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:00:10.338561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:00:10.338561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:10.339582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed0a1ba4-1a1f-4008-a978-2c830fb7e5ff]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:10.339865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:10.341560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:10.344441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd9fe99d-5f00-4e11-8ec2-8473b7a0f9f4]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.344441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:10.344441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:10.344717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:00:10.345045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:10.346424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d48da852-2eb1-4e12-92d8-eee2c27a4e11]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.347343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07881050-dadc-4054-bf67-569bddfe7cb4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.348663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f5cd37e-9270-4339-856e-e6de8e2d0f05]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.369401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57f9d8da-0c60-408d-b0a2-da905d71e47d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 450634, 'reachable_time': 31024, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167726, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.377301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:00:10.377642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f614e835-a5d1-425e-949d-f1fee79e6ccd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:10.378843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:10.381622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:09Z", Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 27, Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 07924206-2ebd-40a2-a99b-548053bfc3a3. Wait time elapsed: 14.815 Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 07924206-2ebd-40a2-a99b-548053bfc3a3. Wait time elapsed: 14.815 Jul 01 10:00:10.393661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:00:13.576072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.195s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:00:13.576599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:00:13.578180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:00:13.578928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:13.579518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:13.580878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48e4725b-15c1-4fa6-924e-f7988dbed6b0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.597378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe1b1fe8-37f8-4813-b7e8-2ef2eb72f0c5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.619988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41dd2615-5342-408d-b049-72553238ef15]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.652815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2cd4de87-bdaf-4b48-b83a-f0c3fc14a6a9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167811, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.673439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f94c3ae-997b-41e6-a548-31825e3723d1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.696210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af8a565c-0755-4a47-8c25-e031181b3c63]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.720758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b48cd2b2-8683-42ed-aa04-d58f8f6efb76]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.747674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecb31730-960c-4be1-b893-042210d3d2e8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.748494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:13.749003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:13.749532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:00:13.749881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:00:13.766623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:00:13.766909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e02aad67-5676-49de-8af9-80e7084f0adc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.769989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf9a2780-cd9c-488c-a79c-2b6755f30bf9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.771020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:13.814778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:13.815841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:13.819442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:13.820027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:13.820594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:13.871199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c28ef795-ff66-4ae9-b494-32b419c2baaf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.906489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[facc8e78-3139-45b0-8bc0-187cf0e5c915]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:00:13.906489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:00:13.906489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:00:13.906489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:00:13.906489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:00:13.906489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:00:13.906489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:00:13.906489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:00:13.906489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:00:13.906489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:00:13.906489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:00:13.906489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:00:13.906489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:13.907530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[facc8e78-3139-45b0-8bc0-187cf0e5c915]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:13.907757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:13.910341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:13.912459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff6edabd-84b8-4f48-b6da-c7f3d4cd6a6d]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.912459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:13.912678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:13.913089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:00:13.913583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:13.914538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7aa0cba4-ec17-4d0b-bd6c-b646ca831f54]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.915573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93301f92-e306-47cd-9770-cb1373caa153]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.916458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e67c03b2-73f9-4a42-8024-d6bccf477540]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.939491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1853927b-e0fa-41a5-9886-5b3a7e7844fa]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 450991, 'reachable_time': 19856, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167829, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.946976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:00:13.947231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89153459-b2fc-43f6-b09f-63f9a59d913c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:13.948461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:13.951123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:11Z", Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 07924206-2ebd-40a2-a99b-548053bfc3a3. Wait time elapsed: 18.386 Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 07924206-2ebd-40a2-a99b-548053bfc3a3. Wait time elapsed: 18.386 Jul 01 10:00:13.969152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:00:14.878408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.927s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:00:14.879502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:00:14.880530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:00:14.880935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:14.881870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:14.883100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0e4d98f-eb00-4758-8393-79ef92223c19]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:14.898366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc10c7aa-bf53-4ec8-83b9-141605cad43a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:14.923883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b98a0767-8d79-4dda-91e4-22e494245d9b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:14.970476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19b07311-d593-4683-be89-54affe14d8a3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167832, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.003979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1eb7259-a050-40ce-a3fe-a0c07086db2d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.029927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0005dc8e-9ed4-40b3-b95b-49f802995fbd]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.052283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa03eab9-b615-460d-97e6-8a7123496448]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.075969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d59cdac4-7101-4734-b337-f8689e27d18b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.077584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:15.077584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:15.077787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:00:15.077994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:00:15.097302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:00:15.098358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46148614-37e4-4ed1-87a4-91337a8c01e3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.101757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45c31053-ea37-4d46-b1a2-5f0a8f91cb52]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.103657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:15.111345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:15.113659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:15.114230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:15.114670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:15.143958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:15.145102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:15.145852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:15.146455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:15.147205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:15.194731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:00:15.201882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f14d2ae-e5c6-4033-ad9f-f1eebfe4d8e5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.253849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7df97c23-0107-4263-93dd-49d88f6ff274]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:00:15.253849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:00:15.253849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:00:15.253849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:00:15.253849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:00:15.253849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:00:15.253849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:00:15.253849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:00:15.253849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:00:15.253849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:00:15.253849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:00:15.253849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:00:15.253849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:15.254838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7df97c23-0107-4263-93dd-49d88f6ff274]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:15.255106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:15.258146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:15.260524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ec7e4bc-7fa0-40ad-b29f-c226b66bb61a]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.260524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:15.260720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:15.260879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:00:15.261362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:15.262441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9bf1b93-9a6b-43e8-bcfe-c28b8858eaef]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.263935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[795b970c-2aaf-4d2d-bc6e-e2b85258fb48]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.266165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f06ab644-8841-4ff0-ba6f-6b891461cfd5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.288804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34513324-6fbb-443d-bd32-3ba8895b1ca2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 451123, 'reachable_time': 39460, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167852, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f260b32-a160-4d21-9dd4-1fc3a1de20ea]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:15.294391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:13Z", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 31, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:15.299973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:15.318619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:15.318619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:15.318619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:15.318619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:15.318619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:15.318619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:15.318619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:15.318619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:15.318619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:15.318619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:00:15.318619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:00:15.318619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3 Jul 01 10:00:15.777838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-3a91ca0a-1597-4250-9611-1b92982e6003 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['4ae39661-aad4-4b8a-8f84-f7daeabb6e80'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:00:15.778853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id a202273f-ed4f-430f-8745-cb87feba9c2e. Wait time elapsed: 0.001 Jul 01 10:00:15.779174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id a202273f-ed4f-430f-8745-cb87feba9c2e. Wait time elapsed: 0.001 Jul 01 10:00:15.779575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:00:18.765537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.986s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:00:18.766143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:00:18.766730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:00:18.767058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:18.767336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:18.768510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc9eb893-0ea1-445b-9495-1e632cf5a3e1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:18.780814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[697f7a62-4a8b-416e-98ab-4ccf7dcb186d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:18.800834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c26cf4bb-5548-4515-ac79-3d97c2db3c87]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:18.829725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a2dde29-3b09-415f-8ec5-17a757f2bc70]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167908, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:18.851696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a49745b-6a11-440c-8d5d-2d7bd9150f31]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:18.872249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7288e130-44af-4cf2-a869-cde2b2b19d07]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:18.894454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16995b42-a30e-41f8-94fc-c48f825a1c43]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:18.914775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[769a9492-2b6f-40f6-83c0-26fc26cc79af]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:18.915345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:18.915647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:18.915908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:00:18.916189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:00:18.931806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:00:18.932116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cba9d234-311d-4b2f-88a9-c76a35708ca5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:18.934857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[037bbd64-064c-4a3d-9320-7f14712841ee]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:18.935923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:18.970902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:18.971633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:18.972290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:18.972775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:18.973243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:19.018096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[542122fc-6e71-48d1-afb8-43950e3f7264]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:19.047517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6a3800aa-f8cf-413c-aca8-17f51f655065]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:00:19.047517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:00:19.047517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:00:19.047517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:00:19.047517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:00:19.047517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:00:19.047517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:00:19.047517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:00:19.047517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:00:19.047517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:00:19.047517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:00:19.047517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:00:19.047517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:19.049581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a3800aa-f8cf-413c-aca8-17f51f655065]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:19.050638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:19.052455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:19.054594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e7db6fe-2f6a-47fc-bb93-90ed8b1ae4c6]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:19.054594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:19.054594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:19.054862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:00:19.055867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:19.057344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bd93f36-2198-4226-ba68-17ff9559a433]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:19.058649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64f2efc9-1cf7-4ca0-98e9-a9a4e1ef6ab2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:19.059770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16e904ad-cc99-4e4b-8479-3388f858ed7a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:19.085652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8713ad7a-9ca0-4b5f-8d3e-1cbe8f1dad7c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 451509, 'reachable_time': 21968, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167926, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:19.090463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:00:19.090463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1516c37-6295-4788-9bcb-fe1b7ab3a80a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:19.092198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "87b82431-0203-44cf-ad59-30f10abab77b", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.106", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "12ee8a9a-ce0b-46dd-8558-fecab39e0b14", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3446, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:1a:5c", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1c056211-3f5e-42ab-a71c-b105366670ad", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:00:14Z", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:14Z", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:00:19.095876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:16Z", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 34, Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id a202273f-ed4f-430f-8745-cb87feba9c2e. Wait time elapsed: 3.316 Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id a202273f-ed4f-430f-8745-cb87feba9c2e. Wait time elapsed: 3.317 Jul 01 10:00:19.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:00:19.893437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.798s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:00:19.894074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:00:19.895442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:00:19.895901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:19.896245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:19.897708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95f562d3-6311-4e17-b2d1-8f45d3ef5e7c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:19.910836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd07c47a-989d-485c-a91a-769ed0d2b813]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:19.934454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d0ef415-e0ba-4ecf-9aea-af52f4187514]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:19.975829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a73e1d17-848c-40af-afbe-9957ff452d61]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167929, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.005320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b7e1642-4f14-485b-9bf4-bb4aeeec9276]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.025194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fc77a1e-dd69-46f0-855c-324091e39353]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.045831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84979dc0-50e3-4e04-ae60-681cf32b6146]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.066734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a19be966-6f4e-4a5f-8e22-645d28d78e57]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.067321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:20.067655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:20.067919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:00:20.068195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:00:20.083187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:00:20.083614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bccef2c5-2b27-4926-9759-7cba6a432f8b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.086130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23f24b6a-0b50-4845-b1cd-0d2a996dd186]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.087066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:20.125455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:20.126219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:20.126831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:20.127445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:20.128076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:20.171980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94a682e6-a619-42dc-8653-bb6768ea243c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.205446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f96000ad-0168-4109-95f7-c563b5ce2575]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:00:20.205446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:00:20.205446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:00:20.205446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:00:20.205446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:00:20.205446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:00:20.205446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:00:20.205446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:00:20.205446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:00:20.205446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:00:20.205446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:00:20.205446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:00:20.205446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:20.206465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f96000ad-0168-4109-95f7-c563b5ce2575]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:20.206670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:20.209398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:20.212204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04fd63a4-9d1b-48d5-9493-7dddbaddf8b3]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.212204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:20.212204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:20.212557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:00:20.212557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:20.213693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e9684c3-e7ee-41e4-9b51-da4371ae97f7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.215266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[593bb662-1fbe-4d53-9e44-f4c62614d87d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.216252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9afcd55f-d9c7-4087-a714-d9b49e7d0d85]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.237322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52261988-30cb-49b9-8a16-2452ca27f4b3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 451623, 'reachable_time': 29452, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 167948, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de683f98-9ae0-4eec-ab3f-2bc464fe938a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:20.242047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "87b82431-0203-44cf-ad59-30f10abab77b", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.106", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "12ee8a9a-ce0b-46dd-8558-fecab39e0b14", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3446, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:1a:5c", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1c056211-3f5e-42ab-a71c-b105366670ad", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:00:14Z", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:14Z", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:00:20.249501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:18Z", Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 35, Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id a202273f-ed4f-430f-8745-cb87feba9c2e. Wait time elapsed: 4.465 Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id a202273f-ed4f-430f-8745-cb87feba9c2e. Wait time elapsed: 4.465 Jul 01 10:00:20.272854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:00:23.028574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.785s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:00:23.029168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:00:23.030990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:00:23.031328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:23.032473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:23.033784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4ea87e4-a5b6-40c6-8806-f818617fa461]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.055529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa62daad-44c5-4b56-b7fa-3dd3061ed4fd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.106506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c0e344c-f691-4ef7-8b85-4700a2dcf575]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.169458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[652b0069-0b18-4f6d-b8b7-a9b234a69681]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 167990, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.200223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66ef6b0c-b434-4f0a-bfc3-92643c46810b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.221558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc1564db-b6ce-49e1-8d5b-5284001581ca]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.245415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c817481e-99e5-435e-ba6e-666df049bc81]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.268223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f16055eb-2aaa-44fe-85ac-ce98593b2589]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.268833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:23.269102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:23.269529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:00:23.269805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:00:23.285368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:00:23.285727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9056312f-6462-42b6-924a-ba9763a20868]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.288504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3079724a-4f0c-46dc-942c-27aafdadd4b2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.289416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:23.317162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:23.319088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:23.319630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:23.320110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:23.320587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:23.361649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b88183e8-f845-408f-97e0-9ffc0d15e3dd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.391994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a745f181-d5d2-4787-9761-c4e9f6f4ce24]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:00:23.391994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:00:23.391994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:00:23.391994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:00:23.391994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:00:23.391994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:00:23.391994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:00:23.391994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:00:23.391994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:00:23.391994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:00:23.391994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:00:23.391994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:00:23.391994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:23.393092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a745f181-d5d2-4787-9761-c4e9f6f4ce24]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:23.393307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:23.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:23.397232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c7a243f-ce7b-4dd6-88e2-95f2dce7cdd9]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.397232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:23.397232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:23.397232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:00:23.397232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:23.398570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21909f75-f2be-458e-a514-8650b02b4b7c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.399583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8344f99d-318b-4cf1-88e0-2a88eaa645a9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.400576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8ace8c2-1f4b-453c-a5fb-5ae5d71563bc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.421321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af5b35fd-97f9-4ffc-b7dc-d39efffdb15f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 451943, 'reachable_time': 31416, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 168018, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.423765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:00:23.424196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82c525e1-9918-4707-8d8a-ea21a71f726f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:23.426980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "87b82431-0203-44cf-ad59-30f10abab77b", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.106", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "12ee8a9a-ce0b-46dd-8558-fecab39e0b14", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3446, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:1a:5c", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1c056211-3f5e-42ab-a71c-b105366670ad", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:00:14Z", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:14Z", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:00:23.429817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:21Z", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 36, Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id a202273f-ed4f-430f-8745-cb87feba9c2e. Wait time elapsed: 7.651 Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id a202273f-ed4f-430f-8745-cb87feba9c2e. Wait time elapsed: 7.652 Jul 01 10:00:23.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:00:24.256473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.826s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:00:24.257035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:00:24.258349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:00:24.258739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:24.259091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:24.260723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55aee85e-40b4-4150-a852-1136fa7958ac]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.273169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba0e54cc-a35c-43af-9c0b-fe48012a01be]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.295055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fbeea71-766a-455c-b3a9-bd4a07ca11e2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.320823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53069523-fd0c-45f6-ba2b-c4e94416ce90]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 168021, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.342876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec1322ad-0f89-4613-b9ad-3ee6c1dc4292]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.362761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc251b0c-87fb-4194-b26b-d9ee4ebe6eb3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.382513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57563c50-4fb6-4296-94f1-32cd53d9e996]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.402495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1546079-547b-45d3-886b-02df108e6382]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.403128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:24.403505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:24.403925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:00:24.404334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:00:24.419796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:00:24.421037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a89b71f8-4050-4772-902e-cb34b1a9b2a6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.424147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64490cf9-2585-494d-8a72-362a3049137a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.425159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:24.462074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:24.462757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:24.463157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:24.467808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:24.468396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:24.510803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79f80cf1-3a0a-450d-8ba1-d94d1fcc66dc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.541405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[99de6ead-9022-4c11-bb6f-534ca52cda44]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:00:24.541405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:00:24.541405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:00:24.541405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:00:24.541405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:00:24.541405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:00:24.541405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:00:24.541405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:00:24.541405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:00:24.541405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:00:24.541405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:00:24.541405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:00:24.541405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:24.542525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99de6ead-9022-4c11-bb6f-534ca52cda44]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:24.542617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:24.544506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:24.547389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ee28e39-b389-44b8-8f2c-50d883df342d]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.547389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:24.547389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:24.547389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:00:24.547389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:24.548171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f05a851-43df-4be8-ac05-7605f76eceaf]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.549378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5927d74b-dcdc-4dc2-95cf-5aa4eb4abafe]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.550289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fff4b29a-bdb0-415b-bfdf-e4dda8fc6614]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.568654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[539b69d1-5953-4022-b989-09d0bde06e3c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 452058, 'reachable_time': 15088, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 168040, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.573809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:00:24.574558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82d1153a-759e-4afe-af61-4a24f1eb6210]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:24.575602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "87b82431-0203-44cf-ad59-30f10abab77b", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.106", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "12ee8a9a-ce0b-46dd-8558-fecab39e0b14", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3446, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:1a:5c", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1c056211-3f5e-42ab-a71c-b105366670ad", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:00:14Z", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:14Z", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:00:24.578652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:22Z", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 37, Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id a202273f-ed4f-430f-8745-cb87feba9c2e. Wait time elapsed: 8.799 Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id a202273f-ed4f-430f-8745-cb87feba9c2e. Wait time elapsed: 8.800 Jul 01 10:00:24.590930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:00:27.190293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.612s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:00:27.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:00:27.192950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:00:27.193508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:27.193794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:27.199217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b79d37b-f603-4aad-8b18-cd63ba376b9c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.237026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5618e88f-45fe-4883-9f36-083600d87fd9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.268665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73a45da3-bad8-4e88-a32b-ada65af6799f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.344254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3094386-8fc5-428a-bb79-28c7ec77ae1d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 168070, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.388362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad81006f-0ce0-4f1a-8941-e0604af1b7aa]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.412248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f14758e-7910-4d1c-bca0-2f0880797eae]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.461505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[712c55b8-11dd-4ce3-b075-45964f9e0929]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.510086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34beb94a-1c56-483b-96e3-5a57f9e55bce]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.510787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:27.511186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:27.511614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:00:27.512603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:00:27.541069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:00:27.541479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f33b7b9-ead7-483d-9e96-70fd5f8f9b8f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.547487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b429c59-a92c-4ad7-92e3-095cc1f9f7a9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.548339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:27.615195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:27.616363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:27.616836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:27.617292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:27.617744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:27.662655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe5af763-186a-46df-b109-a8bca165fd49]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.696231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e7340d56-fc35-4f6a-8df2-dd6675cf1b0a]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:00:27.696231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:00:27.696231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:00:27.696231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:00:27.696231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:00:27.696231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:00:27.696231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:00:27.696231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:00:27.696231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:00:27.696231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:00:27.696231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:00:27.696231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:00:27.696231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:27.697330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7340d56-fc35-4f6a-8df2-dd6675cf1b0a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:27.697438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:27.699317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:27.702500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53bf0549-759c-4888-a037-60ddd6f2cb22]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.702500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:27.702500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:27.702500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:00:27.702500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:27.704466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f59cc5d0-b155-4a67-a39c-407518664cde]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.704466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8452aa3b-3ce0-448a-9b8f-15bcf2149916]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.705534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1e1b899-cbb3-446e-9177-094312099f91]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.725335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d51cc8d8-1ad3-41f3-90d7-d0eb8fc983ef]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 452360, 'reachable_time': 33492, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 168111, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df1be2a4-f849-4423-9d6f-1d40892464f7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:27.730297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "87b82431-0203-44cf-ad59-30f10abab77b", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.106", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "12ee8a9a-ce0b-46dd-8558-fecab39e0b14", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3446, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:1a:5c", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1c056211-3f5e-42ab-a71c-b105366670ad", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:00:14Z", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:14Z", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:00:27.735850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:25Z", Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 38, Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id a202273f-ed4f-430f-8745-cb87feba9c2e. Wait time elapsed: 11.952 Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id a202273f-ed4f-430f-8745-cb87feba9c2e. Wait time elapsed: 11.952 Jul 01 10:00:27.747291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:00:28.463589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.733s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:00:28.464198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:00:28.464794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:00:28.465112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:28.465389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:28.466525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c75a1e8-efa9-4c0a-8128-24308c2ffa77]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.479935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[363e17c6-0f54-494c-8ccd-7100536dd2f7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.502756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1702a10c-c679-43df-842a-8db7e98181b6]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.534998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1e0217d-98f6-4b10-94b1-079e9d68855a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 168114, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.561989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ffe65a7-ce27-4c9e-802c-af78533e8012]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.582489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58fb675d-2bb5-4a42-a3a4-3e84e290eb62]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.594724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c3be36b-4386-42de-8c4d-a6a1a9dadffc]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.615686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bd5a4ab-2e1c-488e-b124-402b89677c0f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.616248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:00:28.616732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:00:28.617039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:00:28.617309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:00:28.632145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:00:28.632486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0acbc6b-95e6-4376-99b6-6f3c5b30d002]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.634979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c65fffea-33cc-4b01-ba89-a4af84c09ade]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.636198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:28.673885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:28.674885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:28.675280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:28.676007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:28.676732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:00:28.707903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eee21398-b16c-4337-8a9a-e68a66c7ce4f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.743331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c371077a-1a35-4f47-aed5-c52744790a4d]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:00:28.743331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:00:28.743331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:00:28.743331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:00:28.743331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:00:28.743331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:00:28.743331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:00:28.743331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:00:28.743331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:00:28.743331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:00:28.743331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:00:28.743331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:00:28.743331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:28.744401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c371077a-1a35-4f47-aed5-c52744790a4d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:28.744977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:28.746663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:28.748824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d95b02d9-0a38-48bc-873d-39a612f2cddc]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.748824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:28.748824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:28.748824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:00:28.748824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:00:28.749564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78e06f16-6200-42f1-ae29-1121ed52eb23]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.750874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ade20fb-8ecf-4c06-8f79-c709bf1fe7f5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.751862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76d77cb7-3a3b-43f2-b08d-fb7e60228c9e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.770955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[259fd827-665d-415e-874f-1c4ed41d0dcf]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 452479, 'reachable_time': 26836, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 168133, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.776339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:00:28.776603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb1d849a-c798-448c-aa23-43a2785983c2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:00:28.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "87b82431-0203-44cf-ad59-30f10abab77b", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.106", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "12ee8a9a-ce0b-46dd-8558-fecab39e0b14", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.4", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3446, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:1a:5c", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1c056211-3f5e-42ab-a71c-b105366670ad", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:00:14Z", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:14Z", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:00:28.782097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:27Z", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 39, Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:00:28.792884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3 Jul 01 10:00:55.199127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:01:15.112189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:01:15.112819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:01:15.112819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:01:15.113002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:01:35.200356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:02:15.113357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:02:15.113973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:02:15.114102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:02:15.114439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:02:15.202642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:02:55.204478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:03:15.113783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:15.114479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:15.114479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:15.114679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:35.207339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:03:47.479159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-1c04b269-375d-4cb6-9220-6a498ed544dc tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['4ae39661-aad4-4b8a-8f84-f7daeabb6e80'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:03:47.480560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id 3b0333b8-22b6-430d-838c-faba146bcedc. Wait time elapsed: 0.001 Jul 01 10:03:47.481116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id 3b0333b8-22b6-430d-838c-faba146bcedc. Wait time elapsed: 0.001 Jul 01 10:03:47.481500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:03:49.684670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.203s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:03:49.685287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:03:49.686105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:03:49.686487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:49.686829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:49.688112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4041469-930d-443b-881c-69632016681a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:49.706870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[576427f5-ee2e-4229-a3ca-57c1fdc8a441]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:49.729834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ba9dfd7-05df-4355-a67b-e54105a0611f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:49.759452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61da5164-bce2-4597-954e-3a1be020721c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171292, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:49.782230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[088ef6aa-2dbb-4ce1-80a6-aef0a53b2314]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:49.802968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bed58b3-ba6a-4343-98d2-01ad6a5de15f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:49.824948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[456d7307-b945-482e-ad3c-162b7427de7a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:49.846385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f443bbd2-8318-4c79-a9cf-c6165442a145]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:49.847128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:49.847492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:49.847824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:03:49.848768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:03:49.870763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:03:49.870942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be658464-f24d-49ac-973c-6ffeac2024fd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:49.873961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e17881aa-4c48-4a1b-99e5-b46d335f2b7b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:49.874988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:49.916155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:49.917256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:49.918129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:49.918784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:49.919575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:49.984069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7327d34a-1c40-4452-a9a4-89c1a2448133]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.024569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[208f6569-8d40-40f3-a939-101b8030affe]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:03:50.024569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:03:50.024569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:03:50.024569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:03:50.024569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:03:50.024569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:03:50.024569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:03:50.024569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:03:50.024569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:03:50.024569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:03:50.024569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:03:50.024569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:03:50.024569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:50.025749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[208f6569-8d40-40f3-a939-101b8030affe]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:50.026558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:50.028673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:03:50.031500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6fad02d8-36c4-408b-877e-2b970751c9ab]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.032053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:50.032490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:50.032813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:03:50.033249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:50.034627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b60bfc1e-a63e-4593-bb02-464519205cc7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.035767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70c3f39e-7914-4430-b018-ae5115b3fb50]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.036841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9782cfe5-92bc-448c-b678-e7fef4c4f7f8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.057222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[638a6de1-1a96-4469-a551-4b77c0c615d5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 472602, 'reachable_time': 21032, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171311, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.059695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:03:50.060142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ea32bf3-e185-495b-9154-b20fd99fb143]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:50.061312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:03:50.067001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:31Z", Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 41, Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 3b0333b8-22b6-430d-838c-faba146bcedc. Wait time elapsed: 2.589 Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 3b0333b8-22b6-430d-838c-faba146bcedc. Wait time elapsed: 2.589 Jul 01 10:03:50.081455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:03:50.838466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.770s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:03:50.839055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:03:50.839867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:03:50.840258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:50.840601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:50.842363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[923b263d-f9eb-490e-8686-46f43909332b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.852497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45aef397-d495-44bc-aac4-12fbfe1a8b8c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.875004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a50f9039-6fa6-4961-8be8-2bba28f439ea]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.903763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[233c7013-b73d-4529-a70f-84466eed7a46]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171314, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.924615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[894ce0ef-962e-4084-86e7-cf9097d9d682]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.946162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3f073a5-8667-48c3-8803-be71b77bf483]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.973708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a7b9d48-8c7b-4c83-a40e-8237b35172a1]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.995619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c747cb1c-f9b9-47f1-a3e4-b98cae84f792]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:50.996185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:50.996458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:50.996720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:03:50.996995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:03:51.014125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:03:51.015320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3ef539c-b287-4e4d-adbe-b961348f7f1d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:51.018912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c7eabab-32f7-4490-9b14-e35a135d3951]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:51.019957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:51.062041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:51.062722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:51.063119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:51.063535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:51.064106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:51.110299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b431e023-d79c-4574-9275-98565ac53ca5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:51.136679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[804d82bc-ca58-47a8-bc19-fb130dc0468e]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:03:51.136679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:03:51.136679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:03:51.136679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:03:51.136679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:03:51.136679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:03:51.136679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:03:51.136679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:03:51.136679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:03:51.136679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:03:51.136679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:03:51.136679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:03:51.136679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[804d82bc-ca58-47a8-bc19-fb130dc0468e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:51.138179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:51.141500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:03:51.145785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca728ff0-af0f-494f-a566-208a47cde02a]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:51.145785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:51.145785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:51.145785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:03:51.145785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:51.145785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa7f7cc9-f05d-4a47-a674-2d34b528763d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:51.145785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff74910b-90f8-4b1d-81cd-411b6d4523bb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:51.147652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[409615d2-0b8b-4349-b15f-26c61a87fef3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:51.166448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c2508ad-bb21-4c2c-8b9a-0588bb7f5d2c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 472716, 'reachable_time': 15864, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171333, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:51.169537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:03:51.169951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0dfea90a-2c0b-46c7-9b9a-0a50b3988746]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:51.172556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8318d333-a4f1-4699-b2a8-c465cd609844", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.53", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4a0974f4-1a2d-4a68-9066-0aafda40a927", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3422, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:72:e8:ec", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "26a2e74e-a3f7-4a4e-b0fe-e1a841879c9d", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:54Z", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:54Z", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:03:51.177117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:00:31Z", Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 41, Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 3b0333b8-22b6-430d-838c-faba146bcedc. Wait time elapsed: 3.698 Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 3b0333b8-22b6-430d-838c-faba146bcedc. Wait time elapsed: 3.698 Jul 01 10:03:51.190356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:03:52.880453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-a06b9b31-0d00-4226-9fb0-30974d01ab89 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['4ae39661-aad4-4b8a-8f84-f7daeabb6e80'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:03:53.980122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.802s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:03:53.980647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:03:53.981727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:03:53.981945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:53.982412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:53.983697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5ddde45-9498-4418-b002-1ff8e638bc1f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.001904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[825457e2-0d57-46ed-833d-36ca3ecc5795]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.023561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[320f4815-cdde-470e-9e12-9ce453a799d9]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.051052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60d8269d-1a01-42ce-9a70-a78cc9c2ab8b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171383, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.064019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e74485b1-8115-4ffa-afa1-aaede938f0bc]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.089982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64776197-95d7-46d3-8f0b-248d90fc3669]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.112060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f794478-63f8-45b2-bb9b-3f29e0253777]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.136323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[999055ef-cc5f-4e09-a1dc-e61362f6af1a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.137030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:54.137397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:54.137754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:03:54.138039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:03:54.153966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:03:54.154765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[535927cc-96d1-4fa7-8120-9f86991ca9bb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.157426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc7c5277-54de-4d79-9913-dec26b2da67f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.158432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:54.193951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:54.195435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:54.196794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:54.197697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:54.198833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:54.255844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00e3ff41-c594-4b69-a0d2-c8c4a9494673]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.295192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[842ed277-197d-407d-82d7-8cfe8e0c56a0]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:03:54.295192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:03:54.295192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:03:54.295192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:03:54.295192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:03:54.295192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:03:54.295192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:03:54.295192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:03:54.295192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:03:54.295192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:03:54.295192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:03:54.295192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:03:54.295192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:54.296440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[842ed277-197d-407d-82d7-8cfe8e0c56a0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:54.296609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:54.300305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:03:54.305337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86059610-f570-4a4c-bda1-92bc8e31fa98]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.307613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:54.308044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:54.308376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:03:54.308942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:54.313149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6a04279-12ba-40de-8fb0-e908a65793da]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.314437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07c27a21-3667-44ac-bb42-e7e33af4c6f7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.315887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94eb9abc-f53d-42b2-8099-fad350eddeda]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.345544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b0536be-658e-451f-8d53-823b82404369]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 473031, 'reachable_time': 36296, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171402, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.354128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:03:54.354764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f50cdbe3-b194-4763-b3f7-09ca1dcddf91]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:54.355834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:03:52Z", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 42, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:03:54.359868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:03:54.374724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id e68ed5f4-87f8-4b3f-adcc-6fa6d2a5d37a. Wait time elapsed: 1.489 Jul 01 10:03:54.374724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id e68ed5f4-87f8-4b3f-adcc-6fa6d2a5d37a. Wait time elapsed: 1.489 Jul 01 10:03:54.374724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:03:55.214739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.844s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:03:55.215355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:03:55.218957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:03:55.219440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:55.219824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:55.221445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d071e489-0df5-4a28-9147-189673581383]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.247535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5c8d67f-6969-4d9b-ac6d-105123a95697]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.280460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5693403-f07f-40d9-b9ea-f61ed1b1c34d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.316407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a8a3644-1e5d-421c-a88f-3be74aab822f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171406, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.342585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7606a461-4912-457a-a632-51025a3409dc]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.363834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[398e3f97-bc71-4d42-a01b-72a71914e9fc]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.387096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f60675b2-0094-4b3c-9cc0-7072c33595ab]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.410898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[120ff982-09f6-4155-b73e-6365aea3fa5d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.412925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:55.413189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:55.414464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:03:55.414641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:03:55.452086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:03:55.453019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df7f0a6c-68a8-4250-894c-0134879c5202]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.456367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55b79838-3f4b-4e4b-9e0d-7973027b9501]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.457658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:55.511951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:55.512995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:55.513549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:55.513878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:55.514263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:55.560760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cab1fce8-fa9a-4c35-b6de-70c9b708e318]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.596792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[df6f2a04-c05b-4347-84d4-ea0fd6241819]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:03:55.596792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:03:55.596792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:03:55.596792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:03:55.596792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:03:55.596792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:03:55.596792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:03:55.596792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:03:55.596792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:03:55.596792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:03:55.596792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:03:55.596792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:03:55.596792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:55.597929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df6f2a04-c05b-4347-84d4-ea0fd6241819]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:55.598407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:55.600454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:03:55.604109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f055c59-40c9-4aed-a11b-0dbc5baf88c3]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.604109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:55.604109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:55.604109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:03:55.604109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:55.605597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17181e5b-e527-4b62-91ca-3d21ea9ab8d0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.606592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2804d624-5a09-478a-80cb-629a1762ecbb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.607661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7708772-49a7-40a7-b6d4-251832a3d892]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.629780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fcc31e3-192e-4585-904e-8fee8db89f46]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 473158, 'reachable_time': 35148, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171424, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.632541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:03:55.633296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f77a9a9-20e4-4fe6-86e3-589bac169f6d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:55.636133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:03:54Z", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 44, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:03:55.639806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 3b0333b8-22b6-430d-838c-faba146bcedc. Wait time elapsed: 8.159 Jul 01 10:03:55.659162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id 3b0333b8-22b6-430d-838c-faba146bcedc. Wait time elapsed: 8.159 Jul 01 10:03:55.659162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:03:58.488235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.849s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:03:58.489212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:03:58.490352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:03:58.492084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:58.493076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:58.494368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2972de11-b129-4711-bbe2-ceda6658018a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.510548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7c5fc63-53a0-472a-a868-177b4da35423]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.542669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b83c114b-48bb-4b59-9576-20f6ddc2b18f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.583957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6760ad00-90f4-474c-a8cf-5fc09cce0722]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171479, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.628851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c831657-b7f9-43f0-baff-77045279061a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.662848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd2bf0cc-4eef-49e6-9931-d8cfd0a375f0]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.691800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82f76b64-461a-4cc6-8903-5023884b25a0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.732280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5081bd7c-bdd7-4698-9ab6-7a2777fbb936]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.733048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:58.733509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:58.733956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:03:58.734416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(8ee8f1eb-61fd-4bc5-9bbb-60a29da08269) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:03:58.755334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:03:58.755696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08339f9c-120e-44ce-aa08-f88639f75b19]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.760908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc3590e2-47fa-4965-a80b-d15b7238969a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.761929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-8ee8f1eb-61, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:58.808667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-8ee8f1eb-61, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:58.810701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:58.811227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:58.811981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-8ee8f1eb-61, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:58.812693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-8ee8f1eb-61, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8ee8f1eb-61fd-4bc5-9bbb-60a29da08269', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6a:4d:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:58.859531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d2cbc9b-c402-4158-90ff-43835d96e108]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.903510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[12e331fe-cd27-4797-af6a-a5961595c30f]: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:03:58.903510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:03:58.903510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:03:58.903510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:03:58.903510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:03:58.903510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:03:58.903510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:03:58.903510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:03:58.903510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:03:58.903510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:03:58.903510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:03:58.903510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:03:58.903510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:58.905304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12e331fe-cd27-4797-af6a-a5961595c30f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:58.906324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:58.908281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:03:58.910622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb1f4787-9d6c-4748-9298-f00acfe31643]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.910622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:58.911565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:58.911565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:03:58.911565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:03:58.912565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b79b0c18-74e3-4fba-b99f-bbd1e2773559]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.914456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cf35028-3744-4ff2-a788-ebc5fee6aaa0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.915754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43fc4199-99d4-4cfb-824e-12863e8b6291]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.940774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3892f5a8-3114-493c-94f1-54f248ca218e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 473484, 'reachable_time': 26472, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171500, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.948753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:03:58.949014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3dc401ad-ccb8-4713-8668-61f7ebe6e977]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-8ee8f1eb-61 not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:03:58.950050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8ee8f1eb-61fd-4bc5-9bbb-60a29da08269", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a59c9af-e8fe-45f5-bb17-f884613de046", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6a:4d:51", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3398, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:35Z", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:03:56Z", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 46, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d51b6f11-1826-4829-b981-d6beebfaa2cd", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:03:58.953461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id e68ed5f4-87f8-4b3f-adcc-6fa6d2a5d37a. Wait time elapsed: 6.070 Jul 01 10:03:58.989124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 2, update_id e68ed5f4-87f8-4b3f-adcc-6fa6d2a5d37a. Wait time elapsed: 6.071 Jul 01 10:03:58.989124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:03:59.232300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-3f8c6e06-1def-4bae-a868-349f51bad31c tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['4ae39661-aad4-4b8a-8f84-f7daeabb6e80'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:03:59.720096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.768s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:03:59.721155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:03:59.722234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (4ae39661-aad4-4b8a-8f84-f7daeabb6e80), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:03:59.722662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:59.723005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:59.724477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b333986c-6d81-4660-8aa2-98be7e103399]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:59.739986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a049a5d-5720-4368-92f7-ebdfec6d8357]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:59.762885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b299b48-c6ab-4e5d-b61c-152fca7559f1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:59.792546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1161419b-848f-4a6b-9140-aae68f79a6e5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171504, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:59.820816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22b4d9c0-3e3d-4ac6-bb01-751581f1c494]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:59.844412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[340363e3-67da-45a7-922b-397b7c92efc9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:59.865788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[980a2f55-7266-461e-90fb-7b2c31f1ff0b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:59.887148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a0af545-2491-4455-8074-aec64dc228e3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:59.887762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:59.888033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:59.888296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:03:59.908758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4f0e21d-dc95-4473-a42c-4797b9c2f01e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 473605, 'reachable_time': 29360, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171510, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:59.909611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:59.909888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:59.910320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:03:59.910769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:03:59.911656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:03:59.911921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:03:59.912218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'dfa0663a-ae09-4b40-8ad1-7e0724c975ac', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:02:28:16', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'device_owner': 'network:router_gateway', 'standard_attr_id': 3396, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.109', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T09:59:32Z', 'updated_at': '2022-07-01T09:59:35Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-dfa0663a-ae), ns(qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:03:59.928262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-dfa0663a-ae not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:03:59.928642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[950db73b-261a-443e-a22c-ea8a9c6a79fe]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:59.931764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a14056ea-d990-4f2f-8cad-f85179010cb0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:03:59.932751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-dfa0663a-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:59.933074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:03:59.933881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-dfa0663a-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:59.934571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-dfa0663a-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:59.935202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-dfa0663a-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:59.935675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-dfa0663a-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:59.935998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-dfa0663a-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dfa0663a-ae09-4b40-8ad1-7e0724c975ac', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:02:28:16'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:03:59.995178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b47cf5b-de47-49dc-ab93-6c5d3792de01]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:00.029867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[716bb10e-298d-4367-a382-877ab8be15eb]: Network interface qg-dfa0663a-ae not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:00.029867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:00.029867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:00.029867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:00.029867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:00.029867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:00.029867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:00.029867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:00.029867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:00.029867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:00.029867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:00.029867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:00.029867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-dfa0663a-ae not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:04:00.030928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[716bb10e-298d-4367-a382-877ab8be15eb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-dfa0663a-ae not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:00.030928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:04:00.031202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:04:00.053384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbab45ad-513d-47dc-8175-5833f67a1b93]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:00.230295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c835a0ee-c080-4f34-b7b0-a22d99d79f8e]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:00.243417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e076d131-06ef-49aa-9480-77e480b9ae7d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:00.377111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-59db6efe-653c-4c63-b6da-d83165e8c676 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got router deleted notification for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:04:00.418225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bac918a3-b944-4f9d-aecd-83be63f5ef14]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:00.419147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:04:00.419812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:04:00.420345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:04:00.420656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80" released by "process_external" :: held 0.510s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-dfa0663a-ae not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-dfa0663a-ae not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-dfa0663a-ae not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:04:00.422203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-dfa0663a-ae not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-dfa0663a-ae not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:04:00.426827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:00.433127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3a6e5be-c092-409f-884f-d11dd271b9da]: (4, ['qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:00.434276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:00.435676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:00.436125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:00.436843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/4ae39661-aad4-4b8a-8f84-f7daeabb6e80.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:00.438941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[372d32d4-b574-41e6-b60f-20b341364a66]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:00.440006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1127092-9eca-40c1-b070-f7edc1fa26af]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:00.441230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec67c41f-90c2-4a17-b30b-0763a10e03ac]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:00.464378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5277fbe0-c816-4dde-97fd-a29eef1591b6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 473605, 'reachable_time': 29360, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171527, 'error': None, 'target': 'qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:00.475063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:00.475324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be416f23-d1fb-44b8-ba16-de1ea71470eb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 4ae39661-aad4-4b8a-8f84-f7daeabb6e80: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-dfa0663a-ae not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-dfa0663a-ae not found in namespace qrouter-4ae39661-aad4-4b8a-8f84-f7daeabb6e80. Jul 01 10:04:00.476800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1778362456", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109" Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:31Z", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:03:58Z", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dfa0663a-ae09-4b40-8ad1-7e0724c975ac", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:02:28:16", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4ae39661-aad4-4b8a-8f84-f7daeabb6e80", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3396, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.109", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T09:59:32Z", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T09:59:35Z", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id 6344991f-04a3-4b1e-89c4-7dd6e3cd7a44. Wait time elapsed: 1.246 Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, action 3, priority 1, update_id 6344991f-04a3-4b1e-89c4-7dd6e3cd7a44. Wait time elapsed: 1.246 Jul 01 10:04:00.480877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:00.668004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.189s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:00.668552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 4ae39661-aad4-4b8a-8f84-f7daeabb6e80 was not found. Performing router cleanup Jul 01 10:04:00.670039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[100af2fe-f9d5-47d7-84c6-fffb32ba87ad]: (4, ['qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:00.672616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:00.673130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:00.673615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 4ae39661-aad4-4b8a-8f84-f7daeabb6e80, update_id 6344991f-04a3-4b1e-89c4-7dd6e3cd7a44. Time elapsed: 0.195 Jul 01 10:04:02.542829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-a6a370a1-4ab7-4260-87ea-aa7b296ae507 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['7535863b-e4eb-4a6c-93c0-fa9e4ff84b09'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:04:02.543770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 1, update_id 69306d42-cd16-4277-b36a-19c70cb977b6. Wait time elapsed: 0.001 Jul 01 10:04:02.544018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 1, update_id 69306d42-cd16-4277-b36a-19c70cb977b6. Wait time elapsed: 0.001 Jul 01 10:04:02.544312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:03.928594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.384s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:03.929641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:03.930766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:03.931182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:03.931596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:03.932910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61d079ba-8c3a-46fd-90c5-8391e19a2e97]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:03.953722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e574405d-2e1a-4455-b14a-f1e4a0ea707c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:03.978259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd8fab05-2d71-4514-9ab3-0332e70e7875]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.017465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e564e2d-166c-4b68-a41f-126509e03f8b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171592, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.042200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c992b34e-83b9-499a-8f50-79860803d069]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.063526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[973b4cf7-399d-430d-9305-dd48a900b19d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.088700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ec80d21-b87e-48e9-9f13-ec7fbaba4660]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.113184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0dce0c6c-60e5-4e6e-8d1a-e357ded9d840]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.113773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:04.114049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:04.114314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:04.136351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4126c683-5ee2-4079-907c-5bc6a72fbf9f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 474028, 'reachable_time': 15500, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171598, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.137405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:04.137820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:04.138304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:04:04.138567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:04:04.139616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:04.139887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:04.140176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'b60f3449-896f-4f55-8d49-0c13843db69d', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:e2:ff:81', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'device_owner': 'network:router_gateway', 'standard_attr_id': 3662, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.215', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:04:01Z', 'updated_at': '2022-07-01T10:04:03Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-b60f3449-89), ns(qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:04:04.159025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:04.166083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6122d258-55c6-4b19-8719-e5371d4d4587]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.170921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[992badcf-170f-48d2-8417-22bedecd73c5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.172154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-b60f3449-89, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:04.172490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:04:04.173265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-b60f3449-89, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:04.175843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-b60f3449-89, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:04.176317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-b60f3449-89, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:04.176901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-b60f3449-89, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:04.177287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-b60f3449-89, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b60f3449-896f-4f55-8d49-0c13843db69d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:e2:ff:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:04.231611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efb7e6b8-943e-4de7-b8c4-a85844add228]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.268923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7ecb9110-8124-4fbb-9506-12f49496119b]: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:04.268923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:04.268923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:04.268923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:04.268923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:04.268923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:04.268923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:04.268923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:04.268923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:04.268923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:04.268923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:04.268923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:04.268923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:04.271633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ecb9110-8124-4fbb-9506-12f49496119b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.272173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:04:04.273533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:04:04.284516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5240308-3309-47f0-886c-7568a9bdb55c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.495177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ccd468c-f92e-4cc6-9151-9c36839248f7]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.508034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[619b3afe-22e0-4914-b128-108b7899c596]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.699929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bbd968b-9bd5-4f5e-bc42-c76a95ee5d41]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.700675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:04:04.701223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:04:04.701728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:04:04.702228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" released by "process_external" :: held 0.564s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:04.704838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:04.708393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:04.716733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5977e38e-b4aa-431c-ad0e-4e6b3094bf25]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.718332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:04.719065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:04.720365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:04.720965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:04.722266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37ca831b-49d4-4271-b001-14b31294b346]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.723324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b2b3e51-3542-41eb-9c92-e9c018eaa4e9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.727811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a45cda02-62bb-46c6-b886-febcd7c97bad]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.751974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42d91b2f-ba25-4ea8-bdee-52eb8e6f4cd3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 474028, 'reachable_time': 15500, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171615, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.755323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:04.755844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82d24398-de79-4fcc-aab1-5028d7824a08]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:04.758234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:01Z", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:03Z", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:04:04.762444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:04.770180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 69306d42-cd16-4277-b36a-19c70cb977b6. Wait time elapsed: 2.219 Jul 01 10:04:04.770180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 69306d42-cd16-4277-b36a-19c70cb977b6. Wait time elapsed: 2.220 Jul 01 10:04:04.770180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:05.477752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-11dd0304-11ea-4471-b807-16f26b9ef8ff tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['7535863b-e4eb-4a6c-93c0-fa9e4ff84b09'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:04:09.054523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.291s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:09.055095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:09.055740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:09.056058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:09.056337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:09.057521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c782804-9ac1-4619-a6ba-942581c10121]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.068968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d68af8bf-1541-4056-83f0-76c470caf91d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.089651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59f529cc-6cbc-43e4-8133-93700ab123ff]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.116493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c911f481-dba8-4fd6-ac81-767a018f928d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171673, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.137053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[698a04f6-eb6a-4c9f-948a-f057436fe2fc]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.156984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6fbceb79-1887-4f2d-8ca6-30af05251b67]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.177294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[073dfbc0-5d43-4809-aec4-0aa065492972]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.197861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5ee4aa6-3eee-4396-a8be-76ea258adf34]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.198395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:09.198668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:09.198932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:09.199206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:09.218117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:09.218877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[921cddc6-8ec9-4f23-b51c-dfc1e3fd58e8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.221767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b31483c1-b82d-4c40-a2b0-c4abbb23a84a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.222763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:09.223283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:04:09.224071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:09.225197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:09.225354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:09.226268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:09.226714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:09.257745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0fd91f5-9570-4d76-a911-158183dbab59]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.293586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a3314bbc-28c2-4247-8fc2-f65c103e562d]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:09.293586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:09.293586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:09.293586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:09.293586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:09.293586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:09.293586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:09.293586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:09.293586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:09.293586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:09.293586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:09.293586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:09.293586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:09.294837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3314bbc-28c2-4247-8fc2-f65c103e562d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:09.294937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:09.296999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:09.299644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[747c1db5-a0f5-4cf8-a73e-0ca23fbd7bec]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.300435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:09.300813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:09.301072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:09.301465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:09.302560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5202b1ef-fa83-464c-90cb-f95b78ac8bfe]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.303498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f8159a2-66e6-4a49-9fe6-af801117f112]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.304407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a21b1851-a3aa-48a3-a9f2-0573513d796a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.323307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74f543e7-66de-4f89-8d6d-4dcfac88f979]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 474538, 'reachable_time': 18560, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171690, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddf583cc-6284-421a-b26a-4cbf9fbfe033]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:09.328386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:07Z", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:09.332723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 1, update_id 59b5a1ec-c3da-4efb-905d-190f8f83c7d4. Wait time elapsed: 3.854 Jul 01 10:04:09.352669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 1, update_id 59b5a1ec-c3da-4efb-905d-190f8f83c7d4. Wait time elapsed: 3.854 Jul 01 10:04:09.352669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:10.002266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.669s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:10.002537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:10.003507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:10.004028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:10.004309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:10.005735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4873c4a2-ad58-4599-a2fc-a546f0ba672f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.018914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87034f70-45e7-481c-a843-003ecf1ba19b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.040053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d2f1ba3-ab4e-4908-849c-c7d70bea5d5b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.070028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a6587ae-c480-4469-b320-85ff8efe9b98]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171693, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.090468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe2474cb-18a4-4601-9ad6-7c9c719c1fde]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.111033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f3330f2-2ba8-4ae7-b712-545411f32258]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.133975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ddb3890-085e-47a2-a9ff-27be139ecfb0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.155438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[478106b4-da86-46db-b116-ec2cda006db6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.156022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:10.156295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:10.156560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:10.156837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:10.171550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:10.171840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b295136-e03a-4863-9163-c1599b6fd3ea]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.174401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50991a2d-6a74-49b9-83a2-174489b631dc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.175330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:10.215862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:10.216769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:10.217169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:10.217617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:10.218197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:10.270073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b0fb175-811c-4fca-87df-13d57e7b593f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.301779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[24e9df5a-51e8-4e31-88e2-4385130f8dbb]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:10.301779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:10.301779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:10.301779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:10.301779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:10.301779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:10.301779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:10.301779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:10.301779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:10.301779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:10.301779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:10.301779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:10.301779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:10.302963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24e9df5a-51e8-4e31-88e2-4385130f8dbb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:10.303485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:10.305946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:10.308419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4c12019-386a-4373-a371-4d4f2cd1d52d]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.308419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:10.308419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:10.308836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:10.309057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:10.310114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5d6ee38-e075-46aa-ad1d-1e6fb3e3f39d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.311060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0c40f02-f2f9-4587-9f68-14267a45810c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.312007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddf070f1-3060-47d0-8f3f-b8103230e688]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.332999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cc01585-6819-4fb6-af27-08bd99c7918d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 474633, 'reachable_time': 20592, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171711, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.335395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:10.335838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3eb23329-3702-4a55-bd85-ef46d60f19af]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:10.337248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:07Z", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:10.341256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:10.356110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 69306d42-cd16-4277-b36a-19c70cb977b6. Wait time elapsed: 7.798 Jul 01 10:04:10.356110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 69306d42-cd16-4277-b36a-19c70cb977b6. Wait time elapsed: 7.799 Jul 01 10:04:10.356110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:12.783237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.440s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:12.783978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:12.784717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:12.785120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:12.785465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:12.787116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71b47394-8afd-4770-a3fd-bd7d02639a9f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:12.821014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10ee2698-4953-4076-ba8f-4cc452b05ca9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:12.846815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94433f43-10c2-4b3a-94e9-a98dd4ad0753]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:12.877918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[331e0233-c7b5-4183-beea-3e4499e5439d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171762, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:12.901671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e748cbf-8e23-4dd8-91b8-622f1eb8bce1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:12.923025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97b79648-7bc1-4dfb-89fd-b0ee01835dc5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:12.944944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef6e0d99-3323-4a71-8168-b5c3880b185a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:12.965633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c95ac1f9-3dbd-4f4c-a673-b6887ea69e70]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:12.966224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:12.966499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:12.966768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:12.967048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:12.983126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:12.983613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[376f758f-0b07-4c76-8e12-2ad75c2852f5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:12.986462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24417043-3ac2-4d5a-b03f-e6337301dd7d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:12.987640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:13.021123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:13.021735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:13.023037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:13.023507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:13.023904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:13.063033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[143b9386-5afa-4711-8be4-dd555301491d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:13.099877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3d7321c9-0029-4924-b338-d9caecf174c9]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:13.099877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:13.099877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:13.099877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:13.099877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:13.099877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:13.099877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:13.099877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:13.099877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:13.099877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:13.099877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:13.099877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:13.099877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:13.101703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d7321c9-0029-4924-b338-d9caecf174c9]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:13.102095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:13.104235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:13.110998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[250a1171-98ef-4e1c-b159-26a3266aaf0e]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:13.111834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:13.112155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:13.112408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:13.112741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:13.113739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcac32cb-c28d-4434-a1bf-5198c6a2e67e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:13.114665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8fa691b5-abbf-430a-9b5a-11b6467d279a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:13.115594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2660a091-eeb2-42ba-bc71-b99bab9dc37d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:13.134502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28ac7923-7816-4357-a481-18f625b81723]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 474914, 'reachable_time': 20952, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171780, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:13.140001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:13.140438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1efbeb22-f7c4-4a2b-85e2-de4bb27847df]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:13.141498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:07Z", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:13.144466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 59b5a1ec-c3da-4efb-905d-190f8f83c7d4. Wait time elapsed: 7.664 Jul 01 10:04:13.160004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 59b5a1ec-c3da-4efb-905d-190f8f83c7d4. Wait time elapsed: 7.665 Jul 01 10:04:13.160004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:13.971880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.829s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:13.972852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:13.974014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:13.974531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:13.975001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:13.977032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6f9af71-b21f-4532-bed8-fc358f2477f1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.004586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aab3a508-12c5-4703-b808-27fbbd52651d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.034704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfc1162f-d8da-4453-9979-9c4e7aea2214]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.064655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[112aa6e5-5a4a-4f04-b80a-6571a4bc87ec]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171783, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.086603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8da353d2-fe36-4263-ab17-5539658d177c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.107924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3706cad8-22b1-4aed-a08c-92aeb4d6007f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.132195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be867f18-07dd-4dd7-8e2b-fdce62914fdd]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.159255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cece9e2c-61dc-4bcf-9016-e89324e49633]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.159873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:14.160152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:14.160421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:14.160704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:14.179632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:14.180617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00d42048-a6eb-4b55-b31b-a6008d7dd1be]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.183622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0de7d99a-05db-4395-8cab-6e642eb03c92]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.185029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:14.220288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:14.221273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:14.221832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:14.222382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:14.224136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:14.274544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0b727bc-c6e0-460b-8248-6d4037147d3b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.311010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a46ebd5e-ef78-46ee-803d-da3b9754020d]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:14.311010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:14.311010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:14.311010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:14.311010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:14.311010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:14.311010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:14.311010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:14.311010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:14.311010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:14.311010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:14.311010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:14.311010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:14.313122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a46ebd5e-ef78-46ee-803d-da3b9754020d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:14.314220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:14.317694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:14.320845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[744f5878-18ef-4c93-a328-3f75be2a2ef6]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.320845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:14.320845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:14.320845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:14.321245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:14.324263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[687b325d-26ca-4509-976d-6534576fd1d2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.325269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ccac562b-bb3a-4f84-bb77-2f89636de613]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.330466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9df0d927-1d3c-4501-828e-3ff9a2477cd0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.359625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[297ce735-c059-4059-93e0-dc6d18d339be]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 475032, 'reachable_time': 31308, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171834, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.370417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:14.370799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ceb4586-5dcd-4757-8b40-49ca1b8748c6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:14.372030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:11Z", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:14.376326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:14.390576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 69306d42-cd16-4277-b36a-19c70cb977b6. Wait time elapsed: 11.834 Jul 01 10:04:14.390576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 69306d42-cd16-4277-b36a-19c70cb977b6. Wait time elapsed: 11.835 Jul 01 10:04:14.390576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:15.114459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:15.115075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:15.115075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:15.115283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:15.209142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:04:16.895220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.517s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:16.895866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:16.896223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:16.897094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:16.898094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:16.899399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0a6b49e-4fe9-48f6-a77b-98c7a8fcaf32]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:16.911084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d618615a-57b6-447d-b0d3-ff4983fb21b5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:16.932797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1990b43-24dd-469b-8de7-371118d9d729]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:16.960837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60b98258-24c5-486a-8222-dc04089b4777]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171888, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:16.981486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[315fe812-acf4-4cee-853a-5d1541aa7905]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.001340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dcc9f7f7-704b-4a2c-a38c-0539bd960459]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.021435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44f2ee77-f1ba-4e69-8723-4069a962db80]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.041879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d40e9ce8-a302-49f1-8562-c6a09e8ba265]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.042791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:17.043070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:17.043337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:17.043716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:17.062071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:17.062529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64f7febd-80cc-4f6c-b052-f2504d817440]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.065309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbf5bb95-a530-4056-b687-1f0b7d35142c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.066256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:17.098086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:17.099256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:17.099953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:17.100254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:17.102469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:17.162515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94c51238-8e68-447b-9d11-3af2a3a7c237]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.196236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f699e366-ac79-45cb-9fe1-d04a3d9d8d2e]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:17.196236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:17.196236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:17.196236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:17.196236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:17.196236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:17.196236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:17.196236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:17.196236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:17.196236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:17.196236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:17.196236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:17.196236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:17.197325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f699e366-ac79-45cb-9fe1-d04a3d9d8d2e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:17.198053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:17.200061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:17.204330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cd35405-7fa1-415b-9a21-0a5d6325b07b]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.205075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:17.205501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:17.208046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:17.208458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:17.209769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df058f01-5e30-421f-aa78-f66c1899ebfb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.210747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7df2b32-8559-489d-beb1-5c666420b0d3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.211792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ca7e066-f3f5-4946-920d-dab9752ddb80]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.232741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aca529d6-5231-47cb-b6a3-90a815153ab9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 475322, 'reachable_time': 18028, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171906, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.239080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:17.239440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78ddff8e-337a-4b97-9cc8-4162f7ce8390]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:17.240533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:15Z", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:17.244604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 59b5a1ec-c3da-4efb-905d-190f8f83c7d4. Wait time elapsed: 11.765 Jul 01 10:04:17.258290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 59b5a1ec-c3da-4efb-905d-190f8f83c7d4. Wait time elapsed: 11.766 Jul 01 10:04:17.258290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:17.936043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.691s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:17.936749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:17.937422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:17.937837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:17.938239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:17.939624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61ae4759-e5f2-4052-b753-d6563ed7b2ac]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.956486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e537ec06-936d-4266-922e-2c0329d6e63b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:17.977894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78cc3021-0bfc-4787-b417-5c130ab7d8bd]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.012659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[279760b9-78b5-47a9-8a2c-36d28713f749]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171909, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.035884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d700561-11dc-4eca-8670-f0be5de06d85]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.064952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf7f4410-5c0b-4957-84bf-f4be2ae46fed]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.112225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e00c0476-5961-4774-b76b-b14880b83d8c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.133067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70fc0bfa-8c5d-4fea-b27d-d3f639789418]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.135719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:18.136213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:18.136486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:18.136768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:18.153595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:18.153957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2a80e9f-1215-4e0e-8145-338acc3b95b9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.156563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27b77730-3d67-4f2f-9452-8d9d456f284d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.157491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:18.198224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:18.199975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:18.200564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:18.201467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:18.202043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:18.244213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8aceaab1-4641-428c-a878-e935568cf503]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.274682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[684a25f7-286f-4f9f-8709-dbb93a481d62]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:18.274682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:18.274682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:18.274682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:18.274682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:18.274682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:18.274682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:18.274682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:18.274682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:18.274682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:18.274682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:18.274682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:18.274682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:18.275906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[684a25f7-286f-4f9f-8709-dbb93a481d62]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:18.276923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:18.279248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:18.281959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44c52d18-9424-4797-a5a6-154947ccd01a]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.281959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:18.281959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:18.282277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:18.282496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:18.283717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a488ca59-c89c-4a5f-920e-0e026b6822a9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.284936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed3c2f4b-7760-4f07-b929-4e67c7f3bc19]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.285819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e0fbd83-a9c3-465d-a1a5-b8644468b0ce]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.305206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d3dc20c-c9b9-40a0-b0c7-aa12c0d2262a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 475427, 'reachable_time': 15240, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171929, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.310451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:18.310689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a91a352b-bd81-474b-b82a-e29f1f1d3c0e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:18.311761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:17Z", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:18.316842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 69306d42-cd16-4277-b36a-19c70cb977b6. Wait time elapsed: 15.770 Jul 01 10:04:18.333284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 69306d42-cd16-4277-b36a-19c70cb977b6. Wait time elapsed: 15.770 Jul 01 10:04:18.333284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:21.189273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.876s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:21.189921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:21.190466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:21.190793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:21.191754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:21.193155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5639bba8-1aa4-4862-88db-78d833b3ff62]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.227830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c79fa4b-1c59-4ccc-836d-0a1dc8422204]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.251102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4f40285-b594-41cc-bc39-0a99240b4e37]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.290194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cf0c1c1-1cad-4131-a6db-c3836f532f6d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 171977, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.317605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22e6cb8f-21cc-449e-9cde-5fd7385b6665]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.343699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d02eaa98-451d-42d0-8ee0-eb2ba1a42dc9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.364827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3986f13e-de76-4f75-b931-64c77391c700]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.386613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd02f59f-88e5-4a3c-8270-fc81bb42a6d0]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.387197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:21.387515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:21.387804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:21.388084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:21.403128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:21.403554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[029dff1a-6991-4889-8f67-3278e424105a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.406278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2b7880a-5423-40d1-8439-05bed84db8cf]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.407327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:21.439917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:21.440589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:21.440982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:21.441367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:21.441746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:21.497815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e464de6-7ede-4614-bc8c-7ef7bc8c1594]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.530097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3fc8979a-53c0-4efe-bd5c-6b673aaa7f24]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:21.530097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:21.530097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:21.530097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:21.530097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:21.530097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:21.530097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:21.530097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:21.530097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:21.530097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:21.530097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:21.530097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:21.530097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:21.531149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fc8979a-53c0-4efe-bd5c-6b673aaa7f24]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:21.531817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:21.533634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:21.538222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36c8e457-2ed0-4d92-a7e9-bf206c2a8138]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.538222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:21.538222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:21.538222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:21.538222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:21.538222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4898733e-5a91-4bb0-a758-56d38a156dd2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.539079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[500f3bf9-ddb1-4842-b58a-b1296891ecb1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.540159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b53be817-96ab-417e-94cc-5df0ba37969d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.565120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c18d369-8c59-47db-a9ee-218c820cd6d2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 475754, 'reachable_time': 28556, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 171997, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.567397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:21.567732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9d7c0da-b544-4bec-9de5-abe7489c1ce3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:21.568785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:19Z", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:21.572431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 59b5a1ec-c3da-4efb-905d-190f8f83c7d4. Wait time elapsed: 16.092 Jul 01 10:04:21.592109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 59b5a1ec-c3da-4efb-905d-190f8f83c7d4. Wait time elapsed: 16.093 Jul 01 10:04:21.592109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:22.316292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.745s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:22.316903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:22.317637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:22.318017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:22.318355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:22.320171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da8a85e1-bc0d-4696-a34b-780d501cb74a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.335128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8edf5d2a-e586-41bb-8398-a2a11a48f039]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.357624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecfba503-5e01-4902-a9f4-a9e1069dd0ab]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.389823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6af8b0e-9b29-4d42-b0ca-6042eb9d55b5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 172000, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.412097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c23e81d2-09e4-4a91-bc8b-90a44935572b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.437334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d6cde77-028e-4f85-b16e-a97b3ba6eb84]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.457685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f54ee5b8-3a8c-4ca1-9e5a-a967e655ec6d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.478112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10d27481-e2da-43cb-a9c8-88c47923e5bf]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.478928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:22.479209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:22.480080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:22.480369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:22.495475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:22.495772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d506a9e0-0cc8-4f76-94df-fc1db03a8e55]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.498571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[552ea96d-e1b4-4724-9b52-236dec2dc03f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.499565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:22.539040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:22.539754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:22.540159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:22.540577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:22.540967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:22.587970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e20ef6e-4233-4310-b665-cfd279c572ee]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.619531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c1b853de-dd9e-4c31-9b6b-f8b9991ae732]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:22.619531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:22.619531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:22.619531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:22.619531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:22.619531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:22.619531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:22.619531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:22.619531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:22.619531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:22.619531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:22.619531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:22.619531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:22.620679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1b853de-dd9e-4c31-9b6b-f8b9991ae732]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:22.620779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:22.622834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:22.626185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d437fdee-31fb-417c-8c73-1cd85a5ce448]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.626185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:22.626185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:22.626185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:22.626185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:22.626185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b7c32a3-2810-45d3-b4aa-5b2236fc3e29]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.626883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58e8a071-c694-4028-8456-a58ce482aec1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.627804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[731ca6a7-8a7f-4a42-b425-0b1030ed9d43]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.647650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad718525-90a9-4c5c-a384-fbcdd8345cc7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 475865, 'reachable_time': 35120, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 172019, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.650016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:22.650439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e493de35-6e07-46b5-ab4f-5be254ab3b7c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:22.651823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:21Z", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:22.656238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:22.669486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 69306d42-cd16-4277-b36a-19c70cb977b6. Wait time elapsed: 20.113 Jul 01 10:04:22.669486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 69306d42-cd16-4277-b36a-19c70cb977b6. Wait time elapsed: 20.114 Jul 01 10:04:22.669486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:25.334539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.677s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:25.335650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:25.337428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:25.337752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:25.338296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:25.340424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b721aaa-dd41-420f-8076-5d6c5efc9288]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.358439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63932f12-2339-427a-a795-7ea4f4210165]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.383535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[599c1cd4-a94f-45e9-84b2-172397988286]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.419250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83492c35-0f89-4bc1-aece-a462557be58b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 172025, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.446450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70c58a1f-042e-444f-a61a-3a6481a124ee]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.472317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef16e0fe-3bef-4e48-a7c6-f6c93a59781b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.499844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83d7ece9-ee33-4496-b024-1601ffa2ebab]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.514413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b5330b0-2e7b-4981-9fa8-96f815075870]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.514985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:25.515571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:25.515868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:25.516162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:25.534052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:25.534617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e339b060-5cff-4471-8d1b-76af0075d6e2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2f22d47-3c2b-4af1-a168-5fe0df1605e1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.550649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:25.591675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:25.593441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:25.594013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:25.594572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:25.595127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:25.648193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0c97ee6-dde5-48ac-b13b-c53b5d343f89]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.678011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[39d8b889-d460-453f-88de-339161ac6343]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:25.678011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:25.678011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:25.678011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:25.678011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:25.678011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:25.678011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:25.678011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:25.678011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:25.678011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:25.678011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:25.678011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:25.678011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:25.679742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39d8b889-d460-453f-88de-339161ac6343]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:25.682850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:25.684812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:25.687877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db0070df-acfa-48ab-8e41-9abb847ac560]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.688728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:25.689059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:25.689312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:25.689695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:25.690752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18dd4593-2850-4f13-9a29-021c0d4e558b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.691882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9d47532-9ea6-48cf-83f9-046ca5b914cf]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.692959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9dec697a-817d-4dfc-a0f6-82fd83e83e7e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.713668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58382f4a-6340-4733-b371-dee38644376b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 476168, 'reachable_time': 42008, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 172043, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e71c29f-f5c9-4612-a9c7-a14e2ba75299]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:25.717995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:22Z", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3 Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 59b5a1ec-c3da-4efb-905d-190f8f83c7d4. Wait time elapsed: 20.240 Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 59b5a1ec-c3da-4efb-905d-190f8f83c7d4. Wait time elapsed: 20.241 Jul 01 10:04:25.723461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:26.436697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.717s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:26.437291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:26.437889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:26.438210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:26.438491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:26.439706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f711927-1221-400b-988a-073102d47d7a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.453660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63336111-a5b0-4d9c-88b8-62df90c385c9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.476693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f160360-696c-4826-adbc-2605be595730]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.528258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bff4fe19-1b01-49a0-9058-a53715fe41eb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 172048, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.565776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b870bc3b-de68-4e3d-90aa-32b0c5941f86]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.609994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5fdbdd1-d1d6-4419-adba-d23a49b4c344]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.656132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbe99c22-a0f0-4016-b202-d959fbdf7a1d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.717899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41f00383-8db3-4fa5-8df1-398413b3364a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.718753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:26.721862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:26.722562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:26.723579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:26.746617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:26.746966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef6e92fe-eb5f-4ac5-8c69-9f03952bfcb6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.750240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ba24236-3466-428d-be71-0e59e7468460]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.751220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:26.821666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:26.823043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:26.823703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:26.824352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:26.824954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:26.862651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b604232f-1cb1-4d1a-bb40-3b46ba35571f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.907508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d8b6b0a3-0aeb-45cf-93e5-025d2a3ea6f0]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:26.907508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:26.907508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:26.907508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:26.907508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:26.907508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:26.907508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:26.907508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:26.907508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:26.907508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:26.907508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:26.907508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:26.907508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:26.909615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8b6b0a3-0aeb-45cf-93e5-025d2a3ea6f0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:26.911043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:26.912942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:26.915202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d1b59f3-bd2a-4d27-ae8f-356ed22d84cd]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.915202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:26.915202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:26.915202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:26.915202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:26.931108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c111a7b9-9bef-44b4-b105-93fe26a90201]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.937340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ed105a1-5b62-4a60-bc35-824e843be8cf]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.938755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d3b90ff-0f24-43f8-873f-7acc9ce9487c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.972341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5dc36bbe-8ac5-42df-bae3-08a7587910ed]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 476278, 'reachable_time': 43508, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 172101, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.978535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:26.978802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ba9d842-0aa7-4be4-a675-758fecb1f807]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:26.979757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:24Z", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:26.982948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 59b5a1ec-c3da-4efb-905d-190f8f83c7d4. Wait time elapsed: 21.503 Jul 01 10:04:27.005289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id 59b5a1ec-c3da-4efb-905d-190f8f83c7d4. Wait time elapsed: 21.503 Jul 01 10:04:27.005289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:29.358945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.377s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:29.359610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:29.361306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:29.361844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:29.362137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:29.364194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32810e82-b742-4ada-b09a-8bc729fb65dc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.379462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5efb1905-fd6d-4d27-a742-202de0bf0f20]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.401471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b814ee8-bc92-42dd-9671-ce9f9c18cb7f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.434884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[356ca1c0-5a81-40bf-812d-34ed580ea13a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 172118, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.456988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70641c59-79ce-4592-9c69-40f4d3f7765b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.480759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c654f55f-447f-4fce-a941-207687da4c01]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.502614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11edeac8-c69e-41bc-8122-ee89b4398c5f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.516680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2bb54fb-cb1a-4577-be2d-debd560883d5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.517466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:29.517781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:29.518107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:29.518412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:29.535973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:29.536985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a8549da-8a51-4a6b-841b-203c57746b4e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.540129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25c3ef57-b49a-4796-a835-7806b1519b0b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.541301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:29.581655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:29.583103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:29.585092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:29.585688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:29.586246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:29.638542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0fd8c56-b9b5-4fcc-8fdc-20269fd0aaac]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.676847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5079f59e-55ce-4236-b755-4b926e326c2d]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:29.676847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:29.676847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:29.676847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:29.676847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:29.676847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:29.676847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:29.676847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:29.676847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:29.676847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:29.676847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:29.676847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:29.676847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:29.678659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5079f59e-55ce-4236-b755-4b926e326c2d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:29.681308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:29.684669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:29.689542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc084e46-c5ff-40d3-9b24-befcec0fb28e]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.690632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:29.691284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:29.691744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:29.692288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:29.694232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d5b7881-919b-40b1-a638-7809abbe31c8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.695446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[585097d0-8d09-4592-bfbf-a702f44abb8a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.696545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[046bc5b5-90a4-4f69-940f-cc79f3a0a7a5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.716045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7feab0cb-e784-4ce4-b408-c035d1505789]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 476569, 'reachable_time': 18604, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 172136, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.722928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:29.723254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df910319-0be5-43c2-ae8a-e5f680cea33b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:29.724279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:26Z", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:29.727844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3 Jul 01 10:04:30.010431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-3ba35bf1-b49e-4db6-a057-fa5cec2b1294 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['7535863b-e4eb-4a6c-93c0-fa9e4ff84b09'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:04:30.011101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 1, update_id ca347336-2c33-4e32-a0e3-a64810f4a046. Wait time elapsed: 0.001 Jul 01 10:04:30.011352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 1, update_id ca347336-2c33-4e32-a0e3-a64810f4a046. Wait time elapsed: 0.001 Jul 01 10:04:30.011683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:30.910189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.898s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:30.911159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:30.912185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:30.913707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:30.913998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:30.915209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c64db2f-7067-4cf0-a1a8-5f8cc2506248]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:30.933528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[445c0216-a230-408b-97bb-2423e2a1972f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:30.956187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba9db3d3-d035-4c6b-8a3c-333e74f1b7ea]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:30.988803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ecec0d7-0001-4bd2-bb14-af1f58536832]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 172139, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.013282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26a8ab77-1b71-43b4-a46c-6218859520e7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.036008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfb81eec-7a87-4c02-aa02-cbdaefc8b664]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.060074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d54a7c46-2185-433c-8285-3dee3e539efa]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.080156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78d16a86-49d9-4198-9847-6b62a14642e8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.080720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:31.080997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:31.081264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:31.081548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:31.096459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:31.097232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57ff7d2a-d510-4594-a880-29de944e989e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.100535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[145532fc-db79-40c0-a19a-c1590372caf7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.101583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:31.134528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:31.135287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:31.135977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:31.136638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:31.137541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:31.190800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84de240c-637e-4f55-9e59-1d2d793d6e66]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.227604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e729123c-4acd-4de3-b114-2a4b5318ed22]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:31.227604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:31.227604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:31.227604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:31.227604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:31.227604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:31.227604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:31.227604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:31.227604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:31.227604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:31.227604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:31.227604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:31.227604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:31.228699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e729123c-4acd-4de3-b114-2a4b5318ed22]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:31.229365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:31.231197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:31.233500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[296d8ec8-ad32-4be7-872a-f619b66ace27]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.233500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:31.233500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:31.233500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:31.233500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:31.234299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cd96629-4726-4758-8d5c-a3732acb804f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.235485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47fb1efb-ff9c-47c1-8ac2-b6d044d7c6ee]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.236678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24523e52-61df-41d1-95c3-b723944516b2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.258699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe1cf3c4-b987-4ed9-bc18-00da51c91141]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 476725, 'reachable_time': 41040, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 172158, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.264372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:31.264372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f12dcdf5-5b7d-4ebe-ab30-a4434e3ca066]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:31.264766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "63df1c25-ddf7-40a2-8192-55c3ce242932", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.242", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "d2d5aa15-1316-4106-a4a1-a32029d1afc2", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3693, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ba:1a:71", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8261ed77-3b2c-4e79-883e-bd4768e08bf7", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:28Z", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:28Z", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:31.268312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:29Z", Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id ca347336-2c33-4e32-a0e3-a64810f4a046. Wait time elapsed: 1.256 Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id ca347336-2c33-4e32-a0e3-a64810f4a046. Wait time elapsed: 1.256 Jul 01 10:04:31.286607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:33.900402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.633s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:33.900975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:33.901592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:33.901917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:33.902293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:33.903535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a935153-74c9-4c7f-85f1-112ffecc286c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:33.916704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0379d1d6-a462-4bfd-bb3b-41f4904b915b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:33.937497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4145ed3d-f248-4723-abda-855ab11abbed]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:33.966664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33acf91c-788f-4591-b670-a9c3e1c7a6a6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 172209, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:33.988068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d552ec6-846e-4100-8a01-a4e5bb1a762e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.008483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d936710-a5da-4fae-888e-52b6a9ac6eb5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.028432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1e526c8-28a8-4beb-9252-85cd59e70004]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.048048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46f7824d-ae28-41a6-9d15-c2d4801bff65]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.048679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:34.048959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:34.049338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:34.049674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:34.064617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:34.064962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a0bfa2a-e8a2-4c0f-9519-0746e755ca5e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.067728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25aa2dba-f011-4213-89ad-3d9fc1ed2dca]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.069289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:34.102228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:34.103103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:34.103590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:34.104220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:34.104671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:34.154069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f77d51b2-9d97-458a-8bf2-80fcd9297cd7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[333257cb-08c7-4439-b249-cf99ee1cda10]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:34.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:34.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:34.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:34.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:34.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:34.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:34.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:34.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:34.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:34.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:34.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:34.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:34.188545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[333257cb-08c7-4439-b249-cf99ee1cda10]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:34.189458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:34.192787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:34.195880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b789d96-d346-48f2-adba-d6f22a4e954b]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.195880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:34.196324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:34.196324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:34.196643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:34.197888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b374b83a-2e6b-4422-b617-36dd8b1984ea]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.198827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[261fb489-d0a5-4a14-840e-5dfaf2bb1cda]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.199756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[079511cb-d77a-4ba9-8520-3ff6a2feccd7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.219209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f50ec2dc-43e5-449e-88e2-a06a8973643c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 477022, 'reachable_time': 40404, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 172227, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.221369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:34.221786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8787cc40-2203-41bd-abd6-0ca78416e97d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:34.225286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "63df1c25-ddf7-40a2-8192-55c3ce242932", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.242", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "d2d5aa15-1316-4106-a4a1-a32029d1afc2", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3693, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ba:1a:71", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8261ed77-3b2c-4e79-883e-bd4768e08bf7", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:28Z", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:28Z", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:34.228675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:31Z", Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id ca347336-2c33-4e32-a0e3-a64810f4a046. Wait time elapsed: 4.219 Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id ca347336-2c33-4e32-a0e3-a64810f4a046. Wait time elapsed: 4.219 Jul 01 10:04:34.242300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:34.991931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.762s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:34.992577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:34.993295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:34.993683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:34.994142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:34.995424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7516f5a-5702-4513-b587-bf273785da2f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.009916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c782f003-de4c-46f1-a9d2-114e2689d11a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.031020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8ce527d-1e38-46ce-bf8f-90a5168159b3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.056998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88a91c18-68c0-4f27-aa59-5124feccb2d9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 172230, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.077197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[778d77eb-8f63-4f7d-946f-7f4b8682a936]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.098067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4774539a-067a-4e17-8af4-20be06ab4548]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.118799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4e0f4a7-fc1f-4baa-bf17-0ae9dd4536b7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.139038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78f16bd8-06d8-4c97-ad98-2ed668cfe01c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.139608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:35.139887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:35.140157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:35.140436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:35.155594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:35.155875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1d05022-6813-4ddd-b66e-4edc9134d395]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.158561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94c9fbd0-448a-470c-a5ce-7ca90ecdc441]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.159696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:35.202582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:35.204213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:35.204388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:35.204942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:35.205550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:35.264236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f6f3756-e7f2-4f54-8fc1-f08c17a75578]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.294031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[939ba21f-3e34-4247-965f-1897b77a3263]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:35.294031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:35.294031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:35.294031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:35.294031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:35.294031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:35.294031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:35.294031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:35.294031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:35.294031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:35.294031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:35.294031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:35.294031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:35.295110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[939ba21f-3e34-4247-965f-1897b77a3263]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:35.295216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:35.297996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:35.300886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87e097f0-d44b-4cc1-8a54-bc8e0726d918]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.300886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:35.300886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:35.300886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:35.300886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:35.302110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65c54678-ab4a-4b5a-b9d5-00da43de4ee5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.303345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[374ea320-27b8-42d4-bb68-d17a2683cea1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.305285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00acf154-b3ea-4346-9442-5c247d468208]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.326140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84cd43b1-4492-4daf-9061-a3c89fca77ae]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 477132, 'reachable_time': 15332, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 172249, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed1b573f-e8e0-4ae4-98e3-00878d66482f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:35.331134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "63df1c25-ddf7-40a2-8192-55c3ce242932", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.242", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "d2d5aa15-1316-4106-a4a1-a32029d1afc2", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3693, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ba:1a:71", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8261ed77-3b2c-4e79-883e-bd4768e08bf7", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:28Z", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:28Z", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:34Z", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:35.336789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id ca347336-2c33-4e32-a0e3-a64810f4a046. Wait time elapsed: 5.321 Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id ca347336-2c33-4e32-a0e3-a64810f4a046. Wait time elapsed: 5.321 Jul 01 10:04:35.349014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:38.056866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.725s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:38.057440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:38.058233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:38.058559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:38.058847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:38.060384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a1c3a8d-d7c1-43b2-ba11-604dbfdb14d2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e32f986-4b17-4d6b-b2d9-bc2720475d59]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.094986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[002576a5-8a77-483b-bf8c-109120a0da27]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.123872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a19f092b-333a-4c85-98fe-543751bc2703]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 172303, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.145058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8848db1b-c6b3-4ef2-8060-bfb188d6fbf9]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.166471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b95671d0-9f94-4d34-9567-9fb0ccb160b5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.187562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86181c09-a32c-4aea-8aed-3cabc51280c9]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.207921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3363315f-f63c-4662-ab6c-786fe0e6e754]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.208462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:38.208735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:38.209597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:38.210146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:38.224346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:38.225266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05dc6d3a-b98b-481b-bf86-3c037afb31ee]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.227982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e279cc2-d11e-4e7a-b174-a31274bf10f5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.228929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:38.257719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:38.259742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:38.260155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:38.260522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:38.260900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:38.309385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[585ad07b-b48d-48e1-b57b-164a738cc54d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.356644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3dddd164-d459-47f3-9188-9602c76aa926]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:38.356644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:38.356644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:38.356644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:38.356644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:38.356644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:38.356644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:38.356644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:38.356644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:38.356644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:38.356644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:38.356644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:38.356644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:38.357825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3dddd164-d459-47f3-9188-9602c76aa926]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:38.359003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:38.361129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:38.364633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26b76974-03e2-4a26-91c1-352bf442bc62]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.364835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:38.365139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:38.365429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:38.365899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:38.366961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96e833f7-7251-4f21-b017-d750396e43cf]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.367914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[556272b9-234f-40b2-8816-3b3b7e2bd0e9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.368977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[531fb349-76fd-4874-8f2f-bcc80f1c6ac1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.393953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6af947b-ae45-4eee-8ec7-305053520c24]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 477438, 'reachable_time': 16756, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 172322, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.402338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:38.402737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff389a02-033d-4e97-9d43-d45e7a00ed82]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:38.403949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "63df1c25-ddf7-40a2-8192-55c3ce242932", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.242", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "d2d5aa15-1316-4106-a4a1-a32029d1afc2", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3693, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ba:1a:71", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8261ed77-3b2c-4e79-883e-bd4768e08bf7", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:28Z", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:28Z", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:38.407408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:35Z", Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 24, Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id ca347336-2c33-4e32-a0e3-a64810f4a046. Wait time elapsed: 8.395 Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id ca347336-2c33-4e32-a0e3-a64810f4a046. Wait time elapsed: 8.395 Jul 01 10:04:38.422186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:39.215342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.809s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:39.215981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:39.216670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:39.217060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:39.217403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:39.218947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4e756e8-c83c-4640-8f23-eb95c3a2e9b8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.233015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82bd3cff-0658-416d-bb2e-85865b59a4f7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.254847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bfd7671-6e74-473e-a782-243c1e729195]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.284316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70c5ff39-ae67-4e44-a948-0da05634121e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 172325, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.304979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3742dc70-a4d8-47ae-ad35-6c24191dfc47]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.324685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef55d0b9-1795-48da-91b0-ef1f4a7f708e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.345295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be893913-6daa-47ad-bc0b-6adffcba285a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.367015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd07d1d2-1fc4-40f5-86b2-31fea83b7224]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.367998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:39.368295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:39.368557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:39.368829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:39.383060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:39.383498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da6190c6-1ce5-44b0-aed3-572cab72c596]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.386151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb5dfa76-d3c0-4fea-93a1-6d829f644b23]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.387145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:39.422542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:39.423695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:39.424062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:39.424603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:39.425323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:39.466676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[825c051e-6475-435e-98b3-e95550cf514a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.502027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d9dc863c-2fd1-4741-820d-a0b2a83f2019]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:39.502027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:39.502027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:39.502027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:39.502027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:39.502027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:39.502027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:39.502027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:39.502027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:39.502027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:39.502027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:39.502027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:39.502027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9dc863c-2fd1-4741-820d-a0b2a83f2019]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:39.503252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:39.505311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:39.509216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48d0e3fb-0536-4454-9b16-7210a18af78c]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.509216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:39.509216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:39.509216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:39.509216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:39.510664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5712228-61da-4e02-a256-ced4bb0ed47b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.511679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bca0dd22-e85c-46f9-b99a-590160812986]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.512655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22ed62a7-d297-4afb-9059-d397d05107c6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.532524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d63bde44-cd36-4453-af03-9780b3db546d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 477554, 'reachable_time': 44760, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 172344, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.536835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:39.536835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74206102-2d38-44b1-ad6d-0cf638389422]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:39.539615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "63df1c25-ddf7-40a2-8192-55c3ce242932", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.242", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "d2d5aa15-1316-4106-a4a1-a32029d1afc2", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3693, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ba:1a:71", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8261ed77-3b2c-4e79-883e-bd4768e08bf7", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:28Z", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:28Z", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:39.543000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:37Z", Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 25, Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id ca347336-2c33-4e32-a0e3-a64810f4a046. Wait time elapsed: 9.531 Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id ca347336-2c33-4e32-a0e3-a64810f4a046. Wait time elapsed: 9.531 Jul 01 10:04:39.558614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:04:42.589283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.048s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:04:42.589919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:04:42.590666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:04:42.591185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:42.591515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:42.593138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd44a03c-d75b-4691-8ec9-9e372d5ddde2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.613403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0dfa6188-bcee-4f01-b810-dd0c0de6eea5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.634216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94beb093-1264-4b21-95d7-72a46d38a357]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.663803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d174ac2-4841-4fa1-8fb8-21095c7d2d79]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 172397, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.684565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0300a1d4-b82c-4cff-a238-16585b2e57f6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.707011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f7db854-ced5-4500-8911-e72b1f0a94c6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.729826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13ff483d-7506-4efe-9432-0584d632f7fc]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.750695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85d39049-ed2c-435b-a5a4-0fa721310b09]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.751183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:04:42.751531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:04:42.751844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:04:42.752204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:04:42.768381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:04:42.768685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bdb8cba-7649-44af-a1f8-9e5563f1e9a1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.771800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb3a73c9-6b94-480c-94ec-f9acbf5a9010]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.772727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:42.810085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:42.811892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:42.812468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:42.813057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:42.813830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:04:42.857339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4840450f-fd4a-49f3-a217-ea30105c93fa]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.890438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4b8256ce-9e26-4082-9402-bb852de6219f]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:04:42.890438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:04:42.890438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:04:42.890438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:04:42.890438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:04:42.890438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:04:42.890438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:04:42.890438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:04:42.890438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:04:42.890438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:04:42.890438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:04:42.890438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:04:42.890438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b8256ce-9e26-4082-9402-bb852de6219f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:42.891719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:42.893987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:42.896924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c883a4a-4b1d-482c-93a7-5c4cd9324bca]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.896924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:42.896924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:42.897224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:04:42.897372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:04:42.898841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a14c8759-0c3f-4516-870e-3435b74721b5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.899902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec4036ef-3974-4782-b3e4-7ed09ab28337]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.900833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f20c6079-b638-4842-848e-92a76db826fb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.919219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b10e6969-2165-40cd-81cc-0b5d1130fdd1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 477892, 'reachable_time': 43512, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 172415, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.924482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:04:42.924929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78d52f79-ff1b-471e-93d0-4b457f8bae95]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:04:42.926044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "63df1c25-ddf7-40a2-8192-55c3ce242932", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.242", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "d2d5aa15-1316-4106-a4a1-a32029d1afc2", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3693, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ba:1a:71", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8261ed77-3b2c-4e79-883e-bd4768e08bf7", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:28Z", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:28Z", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:04:42.931089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:40Z", Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 26, Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:04:42.947077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3 Jul 01 10:04:55.212249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:05:15.115674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:05:15.116449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:05:15.116449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:05:15.116664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:05:35.213344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:06:15.115858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:06:15.116508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:06:15.116508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:06:15.116701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:06:15.215185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:06:55.216711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:07:15.116152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:07:15.116821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:07:15.116821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:07:15.117026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:07:35.218342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:08:15.116327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:08:15.116928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:08:15.116928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:08:15.117082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:08:15.220081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:08:55.221994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:09:15.116410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:09:15.117092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:09:15.117092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:09:15.117260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:09:31.501984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b2980244-f20f-4676-a8da-a3fc9b776331 tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['7535863b-e4eb-4a6c-93c0-fa9e4ff84b09'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:09:31.503241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 1, update_id aa0a2dd7-0cc9-4664-8cc3-d546d983fd28. Wait time elapsed: 0.001 Jul 01 10:09:31.503744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 1, update_id aa0a2dd7-0cc9-4664-8cc3-d546d983fd28. Wait time elapsed: 0.001 Jul 01 10:09:31.504056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:09:32.499600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.995s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:09:32.499873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:09:32.500824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:09:32.501139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:09:32.501591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:09:32.502868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1dd9fb45-72a0-41c8-9055-6ede0e3e1d86]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.520292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8292551-8b4d-492c-9e1b-dd1cb4b6b7c2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.543534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af36672d-f2c9-4142-b444-4a33e22dad15]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.583471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38c0f314-52ac-48cb-84e4-a885d889414f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 177023, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.607433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f62870f5-4089-4d7f-a186-ee751789aaca]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.628771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a80892c-0851-4239-b470-2ddff9e6a3dd]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.650580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0bd67f5f-1371-43eb-abeb-feacd3b637ca]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.676428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d307fba4-1cf1-4ecd-aa08-35743b7400a9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.677125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:09:32.677489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:09:32.677843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:09:32.678213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:09:32.696185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:09:32.696468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b499b3e-a6c0-4de7-8be8-e6b5d63f6b33]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.699985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab20ed48-2121-41fb-be1c-5548bd10783e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.701488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:32.761287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:32.762642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:32.763048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:32.763643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:32.764181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:32.816687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59456445-a3df-4217-85ac-db07e08c8122]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.855061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cdf89be9-2fde-4ebe-a43b-474d099a3e89]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:09:32.855061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:09:32.855061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:09:32.855061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:09:32.855061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:09:32.855061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:09:32.855061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:09:32.855061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:09:32.855061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:09:32.855061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:09:32.855061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:09:32.855061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:09:32.855061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:32.857344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cdf89be9-2fde-4ebe-a43b-474d099a3e89]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:32.858330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:32.859720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:09:32.863473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38c7e2ed-462e-4322-b3a9-f7acff235829]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.864382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:09:32.864706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:09:32.864959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:09:32.865298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:09:32.866295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d562ccc3-4853-4579-97fc-d89941b65703]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.867320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5aabf31b-c2d9-4928-9fc3-7598d7732bac]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.868331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6333665e-4ea0-432b-908c-6099d75ee39e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.890723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eacfcb33-6740-4967-9217-6ed76cfe8839]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 506884, 'reachable_time': 15604, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 177041, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.897429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:09:32.898189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[631d37df-5376-4541-a03a-6ea54c4da301]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:32.900415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:46Z", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:09:32.904783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:09:32.919791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id aa0a2dd7-0cc9-4664-8cc3-d546d983fd28. Wait time elapsed: 1.403 Jul 01 10:09:32.919791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id aa0a2dd7-0cc9-4664-8cc3-d546d983fd28. Wait time elapsed: 1.404 Jul 01 10:09:32.919791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:09:34.768444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.862s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:09:34.769076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:09:34.770013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:09:34.770515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:09:34.771147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:09:34.772731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ba51fef-6518-4186-9a67-454202864089]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:34.791634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c87a310d-7a54-4306-9e4a-078eee883515]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:34.815009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8fc0baff-3976-442c-807f-4e4078b06b57]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:34.850010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eddc8d75-c86b-45d0-a9ad-cbc8a9135595]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 177046, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:34.873191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2201a95-92c3-4d32-a53f-2256da38ae43]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:34.894471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1d17c01-0cc7-4969-bd19-af7254178570]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:34.907719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac401b3a-a854-4e57-94dc-41636f1335ac]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:34.928968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[417060da-7079-48a4-b1ae-f18379f424c0]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:34.929576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:09:34.929850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:09:34.930162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:09:34.930447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(0b7d5362-0fe0-4a18-9b42-c8e64251aae4) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:09:34.947908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:09:34.948168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6298258d-401c-4131-bf83-9a44d673663c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:34.951752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05193776-cbba-477c-b048-345d848282b5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:34.952939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-0b7d5362-0f, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:34.988396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-0b7d5362-0f, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:34.989307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:34.990352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:34.990891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-0b7d5362-0f, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:34.991440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-0b7d5362-0f, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '0b7d5362-0fe0-4a18-9b42-c8e64251aae4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:ee:85:dd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:35.039061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[306227f8-59da-4640-804d-6d350d81baac]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:35.079964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d7c21351-2c08-4181-8fd4-69f31fa102c1]: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:09:35.079964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:09:35.079964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:09:35.079964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:09:35.079964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:09:35.079964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:09:35.079964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:09:35.079964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:09:35.079964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:09:35.079964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:09:35.079964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:09:35.079964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:09:35.079964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:35.081507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7c21351-2c08-4181-8fd4-69f31fa102c1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:35.082658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:35.085219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:09:35.087071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0b39b6e-5407-4070-8f2d-359684b15555]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:35.087340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:09:35.087832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:09:35.088252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:09:35.088595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:09:35.089876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f065478-edda-4887-bfa9-5f07f56623d4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:35.090751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[966d864c-80b7-4cd2-84c8-dd4848cc4d90]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:35.091785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[712e7162-a32e-45d0-af9b-8c25f258cd85]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:35.119491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d257e36-ee6a-4b31-aa7c-857421e5958a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 507111, 'reachable_time': 32740, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 177067, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:35.125616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:09:35.125900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb83afbd-8ac6-40aa-a1ba-8bf33b6744c3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-0b7d5362-0f not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:35.127161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:04Z", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0b7d5362-0fe0-4a18-9b42-c8e64251aae4", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "50a8c7e7-6a3f-4709-8ced-71586663c300", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:ee:85:dd", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3666, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:04Z", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:46Z", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d81273e0-c141-4686-baa8-925682c71bce", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:09:35.129650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id aa0a2dd7-0cc9-4664-8cc3-d546d983fd28. Wait time elapsed: 3.627 Jul 01 10:09:35.150493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 2, update_id aa0a2dd7-0cc9-4664-8cc3-d546d983fd28. Wait time elapsed: 3.627 Jul 01 10:09:35.150493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:09:35.223415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:09:35.974984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.844s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:09:35.975624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:09:35.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (7535863b-e4eb-4a6c-93c0-fa9e4ff84b09), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:09:35.977371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:09:35.978203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:09:35.980000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[adb16b5a-d1fa-450a-99c9-2bcfdd247210]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:35.991346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[625cec2b-1caf-47bc-a94a-39275d16563b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.020132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4154a179-f5ba-43df-884b-c6eb67f9e2a3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.049125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba80f39e-4314-419b-b042-8817b51293a8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 177072, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.070752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88d055d7-eb44-4e11-bf07-34f7f9ce7505]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.094615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c042297-faae-4301-a53b-b15a53a29be2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.116418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[296840b1-1226-455d-8268-a626ec09ed96]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.138534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8576c6a-7e88-4183-a7bb-e8345c7a14c1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.139595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:09:36.140247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:09:36.140812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:09:36.163799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73b232c5-32c2-4714-8eaa-06857ebf6a80]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 507231, 'reachable_time': 17836, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 177078, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.164683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:09:36.165267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:09:36.165727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:09:36.166049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:09:36.166568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:09:36.166986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:09:36.167194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'b60f3449-896f-4f55-8d49-0c13843db69d', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:e2:ff:81', 'admin_state_up': True, 'status': 'ACTIVE', 'device_id': '7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'device_owner': 'network:router_gateway', 'standard_attr_id': 3662, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.215', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:04:01Z', 'updated_at': '2022-07-01T10:04:05Z', 'revision_number': 4, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-b60f3449-89), ns(qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:09:36.184091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:09:36.184374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0b7eda2-50a4-48f4-8c27-57d549e1d6a6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.188850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8bef2e8-c98d-4f17-8516-314f5afe56e1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.190248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-b60f3449-89, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:36.226857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-b60f3449-89, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:36.228008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-b60f3449-89, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:36.228631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-b60f3449-89, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:36.230846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-b60f3449-89, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:36.231980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-b60f3449-89, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b60f3449-896f-4f55-8d49-0c13843db69d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:e2:ff:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:09:36.294114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ee99eb3-a31a-4647-8232-5d1b1b34cca9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.361864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[162a7ca2-1d54-4e35-a4dc-968da43c684e]: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:09:36.361864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:09:36.361864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:09:36.361864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:09:36.361864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:09:36.361864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:09:36.361864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:09:36.361864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:09:36.361864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:09:36.361864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:09:36.361864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:09:36.361864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:09:36.361864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:36.363147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[162a7ca2-1d54-4e35-a4dc-968da43c684e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.363915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:09:36.364585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:09:36.394614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6c4b591-19e2-4085-804a-31bcdeba8bc2]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.458643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-31fcc8cc-984f-4b38-b5c9-045174d16aac tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got routers updated notification :['7535863b-e4eb-4a6c-93c0-fa9e4ff84b09'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:09:36.691188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68c51a19-9431-4bc8-865d-7a007b87f541]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.734487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c5d178d-4e5b-4bf6-a1da-91f112ce534c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.962551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba423d29-6a61-4e8d-879d-f4f5ea43d93d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.963082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:09:36.963425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:09:36.963726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:09:36.964015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09" released by "process_external" :: held 0.798s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:36.965292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:36.967214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:09:36.969497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10d57c33-1485-4509-b5e8-4e4c9f69a2fc]: (4, ['qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.969763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:09:36.970014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:09:36.970277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:09:36.970618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/7535863b-e4eb-4a6c-93c0-fa9e4ff84b09.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:09:36.971710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5cf7a5e-33eb-4755-b2b8-a9c31a531fe1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.972622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45bcd1c5-f78d-498c-bb6a-9387b42a4fc1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.973559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[396f0594-5c06-4261-a2e8-ae787f8f7d9a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:36.993697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6b84b83-d42c-4063-9a22-5e6336764bd5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 507231, 'reachable_time': 17836, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 177143, 'error': None, 'target': 'qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:37.000629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:09:37.001117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c98edd1-a010-44d1-a243-fe813202fc5d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b60f3449-89 not found in namespace qrouter-7535863b-e4eb-4a6c-93c0-fa9e4ff84b09. Jul 01 10:09:37.003596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkBasicOps-router-1213265515", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215" Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:09:35Z", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "70be1bda7f414ebe9c2e0372da0037e2", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b60f3449-896f-4f55-8d49-0c13843db69d", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:ff:81", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "7535863b-e4eb-4a6c-93c0-fa9e4ff84b09", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 3662, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.215", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:04:01Z", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:04:05Z", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:09:37.008146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:09:37.019516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 1, update_id 7b92088f-c103-4321-9c8e-0b5526a2eb15. Wait time elapsed: 0.550 Jul 01 10:09:37.019516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 3, priority 1, update_id 7b92088f-c103-4321-9c8e-0b5526a2eb15. Wait time elapsed: 0.550 Jul 01 10:09:37.019516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:09:37.736699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-9f48840b-0182-407a-84ec-b7e16fa1bc9e tempest-TestNetworkBasicOps-368650648 tempest-TestNetworkBasicOps-368650648-project] Got router deleted notification for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:09:38.384720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.376s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:09:38.385627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 was not found. Performing router cleanup Jul 01 10:09:38.387532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e749b481-35b1-4756-b053-6ddabea7cd38]: (4, ['qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:38.388089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:09:38.388356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:09:38.388647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, update_id 7b92088f-c103-4321-9c8e-0b5526a2eb15. Time elapsed: 1.380 Jul 01 10:09:38.389013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 1, priority 1, update_id b2cb011f-dc94-41f6-9421-1baa26c2b6b5. Wait time elapsed: 0.652 Jul 01 10:09:38.389253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, action 1, priority 1, update_id b2cb011f-dc94-41f6-9421-1baa26c2b6b5. Wait time elapsed: 0.652 Jul 01 10:09:38.389494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09 was not found. Performing router cleanup Jul 01 10:09:38.390629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a217851-86ba-4737-bee8-8aa260fa000a]: (4, ['qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:09:38.391139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:09:38.391447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:09:38.391668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 7535863b-e4eb-4a6c-93c0-fa9e4ff84b09, update_id b2cb011f-dc94-41f6-9421-1baa26c2b6b5. Time elapsed: 0.003 Jul 01 10:10:15.117590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:15.119240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:15.119240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:15.119240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:15.225567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:10:38.601376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-17aff79f-3631-445c-bf87-ae4ff4d3aaf0 admin admin] Got routers updated notification :['893e74e1-66a6-40a4-91fa-1a8006c88697'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:10:38.602943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 1, update_id 07156416-3594-486a-b1a0-646493cd7704. Wait time elapsed: 0.001 Jul 01 10:10:38.603252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 1, update_id 07156416-3594-486a-b1a0-646493cd7704. Wait time elapsed: 0.001 Jul 01 10:10:38.603725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:10:39.966175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.362s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:10:39.967329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 893e74e1-66a6-40a4-91fa-1a8006c88697 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:10:39.968525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:10:39.970010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:39.970507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:39.972044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6905eef8-19ff-4b0c-9ac0-42090a958316]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.000660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6a9d965-5161-4929-bbb4-eee62b910c08]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.027717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93f38469-a808-4f31-88a6-5bb46c149773]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.064387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94ea5694-dfb8-4a98-a17b-c4fa7f06008d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 178029, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.086207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[281d24e9-ad36-468b-bc80-123826d4bfb3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.113320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7a39bae-709a-4a95-bb6e-03bd2a6a065a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.136105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0da991a9-bbe4-4ce3-8910-2037e9138856]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.156848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcf393f8-d668-47ef-b026-04750a614d09]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.157537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:40.158815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:40.159266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:10:40.186778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e775e8a-b554-4bd6-984c-51e862d69f30]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 513632, 'reachable_time': 23144, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 178035, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.187614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:40.188094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:40.188913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:10:40.189537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:10:40.190171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:40.190730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:40.191378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '17c5b8ea-ceb1-467c-a844-03b50c15b9cd', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:7a:1f:c1', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '893e74e1-66a6-40a4-91fa-1a8006c88697', 'device_owner': 'network:router_gateway', 'standard_attr_id': 4082, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.100', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:10:37Z', 'updated_at': '2022-07-01T10:10:39Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-17c5b8ea-ce), ns(qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:10:40.211213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-17c5b8ea-ce not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:10:40.211960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d093d8dc-7c36-4944-9f3b-3b5a5201979d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.218130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d837ed7b-2595-43f1-a0e1-402e79b76075]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.220247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-17c5b8ea-ce, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:40.221037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:10:40.222240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-17c5b8ea-ce, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:40.223257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-17c5b8ea-ce, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:40.223888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-17c5b8ea-ce, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:40.224237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-17c5b8ea-ce, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:40.224631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-17c5b8ea-ce, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '17c5b8ea-ceb1-467c-a844-03b50c15b9cd', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:7a:1f:c1'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:40.277852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[786ae19d-1fc6-4194-ba7d-2bdf6255909a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7f1841bc-0e9c-460c-9679-7aca2559352d]: Network interface qg-17c5b8ea-ce not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:10:40.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:10:40.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:10:40.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:10:40.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:10:40.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:10:40.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:10:40.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:10:40.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:10:40.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:10:40.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:10:40.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:10:40.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-17c5b8ea-ce not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:40.310891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f1841bc-0e9c-460c-9679-7aca2559352d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-17c5b8ea-ce not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.310891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:10:40.311335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:10:40.333365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7242c7fd-907e-4ec2-a949-c6364e4fe37d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.367327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-6f3de61b-45a2-4e9e-8115-52edf3226682 admin admin] Got routers updated notification :['893e74e1-66a6-40a4-91fa-1a8006c88697'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:10:40.515708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a19bed40-fef2-407d-bab5-8e1238539ddc]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.529751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7be5394-1265-4b8f-9ecc-67dfb5ff172d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.734323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[363346db-f3cd-42bd-8ece-455eaaea845a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.735284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:10:40.736198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:10:40.736771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:10:40.737366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" released by "process_external" :: held 0.548s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-17c5b8ea-ce not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-17c5b8ea-ce not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-17c5b8ea-ce not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:40.738324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-17c5b8ea-ce not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-17c5b8ea-ce not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:40.748446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:40.753128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9597df42-d5fe-4a60-8caa-735cdb99c026]: (4, ['qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.754155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:40.754488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:40.755137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:10:40.755137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:40.756709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[011fb8d6-18ea-4eeb-a178-9dea49711178]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.758450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15d529af-c7a0-4d16-aa60-817b0821119f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.761753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41b942eb-cc05-45d5-aa08-47051b2458ad]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.807846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23157f7e-2599-446a-a88a-bc0c15937644]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 513632, 'reachable_time': 23144, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 178057, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.819879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:10:40.820502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5db811e-c356-4f88-8815-52c80dfea9ff]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-17c5b8ea-ce not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-17c5b8ea-ce not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:40.823590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-DeleteServersTestJSON-74730730-router", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100" Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:36Z", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:37Z", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7a:1f:c1", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4082, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:37Z", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:39Z", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:10:40.828621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:10:40.841287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 1, update_id 849d8323-a39b-485f-b6ca-0b561268cc22. Wait time elapsed: 0.471 Jul 01 10:10:40.841287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 1, update_id 849d8323-a39b-485f-b6ca-0b561268cc22. Wait time elapsed: 0.472 Jul 01 10:10:40.841287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:10:45.403141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.563s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:10:45.403818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 893e74e1-66a6-40a4-91fa-1a8006c88697 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:10:45.404903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:10:45.405388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:45.405679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:45.408574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3a26fb1-2118-42eb-8a14-11fba3da4b48]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.422001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a02d2bf5-23be-495d-8b2e-8da23f866d93]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.457088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bc65a6e-45eb-4414-be9b-423d69ffa051]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.488341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f0e0a51-b9d3-4f70-847c-54ff691aed3e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 178113, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.519114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72f14e5b-b996-4627-ac94-877d0055264e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.549039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fda8ec6f-c7b7-4f02-b628-b9a8c5eae4b3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.612052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f60fc3cc-516d-4641-be27-e53efe3d83ed]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.634007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[368144ba-8b1c-4a1f-83d5-daf72bde8ce9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.635075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:45.635530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:45.635652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:10:45.635904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd38837b-22e9-45db-8d27-924c76e385db) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:10:45.672752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:10:45.673105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aed04143-4bae-46ea-97b1-4133d3c01b4c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.676029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4343cfd5-0e60-4ef6-97d5-2c729ba0a823]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.676990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd38837b-22, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:45.677929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:10:45.678865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd38837b-22, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:45.679503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:45.679895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:45.680276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:45.680694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd38837b-22, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd38837b-22e9-45db-8d27-924c76e385db', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3f:9f:bb'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:45.755501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89b2413e-5ad9-4cdb-a965-b23bfd3096b3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.790580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4e661c9f-4366-49e0-abc4-f10073233033]: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:10:45.790580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:10:45.790580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:10:45.790580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:10:45.790580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:10:45.790580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:10:45.790580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:10:45.790580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:10:45.790580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:10:45.790580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:10:45.790580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:10:45.790580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:10:45.790580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e661c9f-4366-49e0-abc4-f10073233033]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:45.791763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:45.793622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:45.796055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63f53ef7-ff1c-47fc-90a5-9573fa7d7e4a]: (4, ['qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.796055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:45.796055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:45.796055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:10:45.796055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:45.796486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1608c2e1-a76b-46b7-9cb0-ba224b7869a9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.797376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[069292d9-39da-41c6-a274-f004e4fe2bb6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.798482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6463009-20f1-469a-a3ef-5957a565714e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.833724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b2405e1-d2d2-4cd1-b85b-b0e8575573bb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 514175, 'reachable_time': 42380, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 178173, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.840596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:10:45.840862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98ab07f4-ac92-4d8e-90e0-0e0d4c411372]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:45.841879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-DeleteServersTestJSON-74730730-router", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100" Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:36Z", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:39Z", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7a:1f:c1", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4082, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:37Z", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:41Z", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd38837b-22e9-45db-8d27-924c76e385db", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a9a659de-f6d1-42d7-9d38-bd056dfcb759", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3f:9f:bb", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4086, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:39Z", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:43Z", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:10:45.844722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 07156416-3594-486a-b1a0-646493cd7704. Wait time elapsed: 7.241 Jul 01 10:10:45.853952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 07156416-3594-486a-b1a0-646493cd7704. Wait time elapsed: 7.241 Jul 01 10:10:45.853952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:10:46.727610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.884s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:10:46.728334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 893e74e1-66a6-40a4-91fa-1a8006c88697 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:10:46.729320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:10:46.729655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:46.730036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:46.731407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d53abba9-04e3-47b1-9803-a821380d6da4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:46.765936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5646b3a3-855f-4070-a03a-fce6242b4c2b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:46.791600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbcffefb-37fd-48a7-a976-86489d847761]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:46.824834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2cd52662-84fd-4f72-a917-8fd120f5f6bb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 178181, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:46.849320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2952b9fb-f376-469d-a84b-6ef858fffdac]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:46.870082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e368d6c-112a-4c1e-aeba-627a7f7aed1d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:46.890123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7349f7d6-fda7-465d-a7c3-3e04cf8882b4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:46.909826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec5e90ad-fd8b-4412-ba80-2981521f9795]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:46.910420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:46.910852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:46.911118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:10:46.911487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd38837b-22e9-45db-8d27-924c76e385db) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:10:46.928108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:10:46.928969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e19e3cc-469b-429d-a8a0-4ee7e2d490b2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:46.932468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59553989-666b-48e5-854f-98e099a18ba7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:46.933430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd38837b-22, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:46.963124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd38837b-22, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:46.964072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:46.964660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:46.967127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:46.967776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd38837b-22, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd38837b-22e9-45db-8d27-924c76e385db', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3f:9f:bb'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:47.018572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9426be0a-1790-4dc5-b65a-f245b96e7010]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:47.051207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[45fcfc34-a5dc-4f2a-b0a2-cfeb43e2a954]: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:10:47.051207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:10:47.051207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:10:47.051207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:10:47.051207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:10:47.051207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:10:47.051207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:10:47.051207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:10:47.051207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:10:47.051207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:10:47.051207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:10:47.051207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:10:47.051207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:47.052124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45fcfc34-a5dc-4f2a-b0a2-cfeb43e2a954]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:47.052412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:47.054078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:47.055979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27351d7c-2626-434a-9c66-e75ba26241af]: (4, ['qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:47.055979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:47.056775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:47.057124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:10:47.057792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:47.058851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9052b76c-74bf-4bf3-b7db-00180021fc56]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:47.059844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9fa64924-7834-4242-83a6-49f03c31a6d1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:47.060760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8f22db3-7c62-41e6-b318-60c3148798f7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:47.080707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51365328-e455-4424-9147-a08a60530524]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 514308, 'reachable_time': 15352, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 178199, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c94dc74-75c1-4962-b2a1-20267f60ce9d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:47.084943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-DeleteServersTestJSON-74730730-router", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100" Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:36Z", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:39Z", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7a:1f:c1", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4082, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:37Z", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:41Z", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd38837b-22e9-45db-8d27-924c76e385db", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a9a659de-f6d1-42d7-9d38-bd056dfcb759", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3f:9f:bb", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4086, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:39Z", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:43Z", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:10:47.090014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 849d8323-a39b-485f-b6ca-0b561268cc22. Wait time elapsed: 6.718 Jul 01 10:10:47.101080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 849d8323-a39b-485f-b6ca-0b561268cc22. Wait time elapsed: 6.718 Jul 01 10:10:47.101080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:10:49.450477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.364s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:10:49.451049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 893e74e1-66a6-40a4-91fa-1a8006c88697 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:10:49.451847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:10:49.452926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:49.453626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:49.455197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e25ad5af-0a0b-4c18-bd3a-805fc7e4016a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.469241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[468b6b98-ab0f-4d0e-a2bc-e19419a564c7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.491129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34f56cd3-170f-4a5f-80d0-1377814f7a4e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.523575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[770cae25-c4b2-46af-8977-c3e26aa034a5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 178202, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.544777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef8afde7-598e-46bb-b310-bfd01341a5fd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.566287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b4778d1-5056-4b07-a086-6b3a2f875c78]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.587928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de733639-0587-44a5-bd5d-510aec640889]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.608222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2fe59e7-8b25-498b-a238-27de1731668f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.609026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:49.609317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:49.609583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:10:49.609858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd38837b-22e9-45db-8d27-924c76e385db) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:10:49.628117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:10:49.628819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8bce956-5ada-4f8a-9222-d5ed582d79d2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.632076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44850a12-6786-459a-abc8-094040c750ee]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.633265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd38837b-22, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:49.672289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd38837b-22, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:49.673460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:49.674015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:49.674661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:49.675354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd38837b-22, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd38837b-22e9-45db-8d27-924c76e385db', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3f:9f:bb'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:49.727054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[872d00aa-7639-4aa5-a03f-2b0a90bfc5bd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.772767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6fd117ad-41e4-4da6-b35e-51b900f13c7f]: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:10:49.772767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:10:49.772767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:10:49.772767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:10:49.772767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:10:49.772767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:10:49.772767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:10:49.772767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:10:49.772767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:10:49.772767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:10:49.772767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:10:49.772767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:10:49.772767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:49.773763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6fd117ad-41e4-4da6-b35e-51b900f13c7f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:49.775190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:49.776939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:49.779192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f4847bc-a987-4f26-b761-8a132d24b256]: (4, ['qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.779192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:49.779192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:49.779192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:10:49.779192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:49.780614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4c5dc8a-3054-4c8b-b5a7-267738308931]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.781966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0bf7d53b-2cd3-42d5-a65e-af2ccd17a4b6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.784168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a7781a5-eea6-440a-ad7b-b522dd8b4454]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.816980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b617bda0-a0f3-4a05-bc7c-25e863599169]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 514578, 'reachable_time': 26696, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 178221, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.819139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:10:49.819410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2caadb60-20fe-4314-a2b5-6cd03ac975f2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:49.820415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-DeleteServersTestJSON-74730730-router", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100" Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:36Z", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:39Z", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7a:1f:c1", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4082, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:37Z", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:41Z", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd38837b-22e9-45db-8d27-924c76e385db", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a9a659de-f6d1-42d7-9d38-bd056dfcb759", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3f:9f:bb", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4086, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:39Z", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:47Z", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:10:49.823718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 07156416-3594-486a-b1a0-646493cd7704. Wait time elapsed: 11.219 Jul 01 10:10:49.833357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 07156416-3594-486a-b1a0-646493cd7704. Wait time elapsed: 11.219 Jul 01 10:10:49.833357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:10:50.537335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.714s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:10:50.537973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 893e74e1-66a6-40a4-91fa-1a8006c88697 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:10:50.538944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:10:50.539455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:50.539863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:50.541866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aaaa8b91-e50d-42f8-a49d-3b1d109f61b8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:50.557024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2765657-1bae-447d-a194-bf9a75c30089]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:50.569584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b4661f6-17d8-4885-8bd8-4f1a2cf13b36]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:50.603814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aca7b825-7db7-4e30-a5b7-c259bd022aac]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 178229, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:50.662091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a02ee57-2eb7-4f93-91c9-22296b407462]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:50.700327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c3f015e-2000-47f7-85d4-155b4c3124d6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:50.730086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0482443e-751d-44cf-bca9-189f31cdb4b5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:50.763079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a03a083c-6aae-4273-815a-6d6c9424981f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:50.764898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:50.765324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:50.765732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:10:50.766170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd38837b-22e9-45db-8d27-924c76e385db) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:10:50.784852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:10:50.785290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0946cd29-0550-4f49-a0ce-6fa96ee061b1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:50.790245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5b8e61f-1f63-4b26-9a18-63cd8faa8a94]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:50.794663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd38837b-22, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:50.869164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd38837b-22, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:50.870931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:50.873950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:50.875287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:50.876479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd38837b-22, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd38837b-22e9-45db-8d27-924c76e385db', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3f:9f:bb'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:50.967670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e2b4d95-f76a-4241-9d86-7afb1cacacf4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:51.016366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f876daf8-fad4-4f55-8749-91fc2c61589f]: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:10:51.016366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:10:51.016366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:10:51.016366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:10:51.016366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:10:51.016366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:10:51.016366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:10:51.016366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:10:51.016366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:10:51.016366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:10:51.016366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:10:51.016366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:10:51.016366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:51.017760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f876daf8-fad4-4f55-8749-91fc2c61589f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:51.019133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:51.021829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:51.030245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69104b69-a78d-4258-aec1-2a1d88a4f1d2]: (4, ['qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:51.031691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:51.032242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:51.032675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:10:51.033240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:51.034473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43157ce9-28b1-4c8d-abb8-f32169c77d82]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:51.035531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[876e56f7-f568-4b5d-b94f-a27c532c4e31]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:51.036627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[696955a0-e73e-4f93-b81d-e9e710095629]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:51.060853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cfb1e9c-9cd8-4dc3-aa1c-57d170ede305]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 514686, 'reachable_time': 29796, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 178290, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:51.070158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:10:51.070417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2191d64c-d250-45fb-80b9-52b112780d3e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:51.072048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-DeleteServersTestJSON-74730730-router", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100" Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:36Z", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:39Z", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7a:1f:c1", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4082, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:37Z", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:41Z", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd38837b-22e9-45db-8d27-924c76e385db", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a9a659de-f6d1-42d7-9d38-bd056dfcb759", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3f:9f:bb", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4086, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:39Z", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:49Z", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:10:51.076668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:10:51.088496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 849d8323-a39b-485f-b6ca-0b561268cc22. Wait time elapsed: 10.709 Jul 01 10:10:51.088496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 849d8323-a39b-485f-b6ca-0b561268cc22. Wait time elapsed: 10.710 Jul 01 10:10:51.088496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:10:53.394839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.317s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:10:53.395436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 893e74e1-66a6-40a4-91fa-1a8006c88697 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:10:53.396050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:10:53.396369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:53.396646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:53.398377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6e9039a-9279-4717-9300-bec0812a7e52]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.425254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3dd966ab-bce0-4dff-81bf-5622ed9bf5b0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.457018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc6b04c2-b86e-4095-beeb-3c52265722f3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.493053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89a4dfaa-d67d-4c34-ab4f-e66b8f739d8e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 178295, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.516384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[506c27e3-838f-40d8-bb47-8a34c3a917c3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.540338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54c1f8c7-42a1-4c8e-8ce8-8972fb86e31e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.566816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04e308a5-10eb-4a29-840c-e5f43b2f5ced]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.591004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e425bc8b-3ea5-4bb6-9284-e4ba70bdfc0a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.591716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:53.592230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:53.592568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:10:53.592932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd38837b-22e9-45db-8d27-924c76e385db) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:10:53.610061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:10:53.610579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cfa3ad19-3374-43ac-ba98-4f0fab988b43]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.613499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[863aee46-96a6-46f7-8b63-21592806e80d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.614555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd38837b-22, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:53.649800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd38837b-22, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:53.650497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:53.650943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:53.651562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:53.652143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd38837b-22, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd38837b-22e9-45db-8d27-924c76e385db', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3f:9f:bb'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:53.697847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06274001-b862-4866-8f87-4c4359f8df63]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.740495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ff6641a4-ccde-4914-aa40-17cf84f4d412]: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:10:53.740495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:10:53.740495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:10:53.740495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:10:53.740495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:10:53.740495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:10:53.740495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:10:53.740495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:10:53.740495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:10:53.740495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:10:53.740495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:10:53.740495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:10:53.740495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:53.741617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff6641a4-ccde-4914-aa40-17cf84f4d412]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:53.741817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:53.745167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:53.748334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d2d3496-36a4-477a-8c7f-cf8d76a368e3]: (4, ['qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.748334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:53.748334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:53.748334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:10:53.748334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:53.751217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a3d05c4-ceba-4dbb-b249-4a590c2c0e91]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.753365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cccd2758-d0b0-4fe9-bc15-28786bcdfdfe]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.756050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5dd54915-6b1f-4c9e-a323-004fe82625df]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.780707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25ece28b-f193-4440-a696-02227a5a5805]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 514975, 'reachable_time': 38460, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 178314, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.782893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:10:53.783403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[088c998e-3862-4a21-9b3e-b7b6b127f8a3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:53.784585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-DeleteServersTestJSON-74730730-router", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100" Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:36Z", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:39Z", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7a:1f:c1", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4082, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:37Z", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:41Z", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd38837b-22e9-45db-8d27-924c76e385db", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a9a659de-f6d1-42d7-9d38-bd056dfcb759", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3f:9f:bb", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4086, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:39Z", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:51Z", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:10:53.789373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:10:53.802839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 07156416-3594-486a-b1a0-646493cd7704. Wait time elapsed: 15.194 Jul 01 10:10:53.802839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 07156416-3594-486a-b1a0-646493cd7704. Wait time elapsed: 15.194 Jul 01 10:10:53.802839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:10:54.842230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.045s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:10:54.842780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 893e74e1-66a6-40a4-91fa-1a8006c88697 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:10:54.843444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:10:54.844012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:54.844499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:54.846488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f3e806e-2324-41f6-a825-915383b742cf]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:54.872975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19c8b7f5-7d04-438d-ac69-c483a147cf4d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:54.896029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f986426-b4ab-4571-ae00-177e8c02d19d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:54.929954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdc66a37-3e21-4c19-8dca-b9328d726f0a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 178317, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:54.954091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3e358b1-e5e5-4a6d-b539-4af58662e18a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:54.979205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c112870-a379-4483-b090-39d6d8407da1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:55.002361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0269888-c021-4ced-abe0-8f3c76ba2b8e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:55.029011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71c4e27d-0f2b-466d-bff2-dc3a2988252d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:55.030087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:55.030397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:55.030666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:10:55.031862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd38837b-22e9-45db-8d27-924c76e385db) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:10:55.049844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:10:55.050219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1efecb99-8af9-4e0c-8077-086283916daf]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:55.053040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63352a94-2038-4f96-b469-7995a8fde2b7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:55.053983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd38837b-22, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:55.092013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd38837b-22, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:55.094606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:55.096444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:55.097933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:55.099200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd38837b-22, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd38837b-22e9-45db-8d27-924c76e385db', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3f:9f:bb'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:55.187656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d32bdb2-7fd5-4551-a555-b1d7c458c71f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:55.225436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[39f32b13-c74f-412a-ae3d-70b16328aec6]: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:10:55.225436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:10:55.225436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:10:55.225436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:10:55.225436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:10:55.225436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:10:55.225436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:10:55.225436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:10:55.225436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:10:55.225436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:10:55.225436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:10:55.225436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:10:55.225436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:55.228135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:10:55.229441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39f32b13-c74f-412a-ae3d-70b16328aec6]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:55.230842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:55.233862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:55.236711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0e04448-1aa7-439c-bf13-3609998b0cbc]: (4, ['qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:55.238073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:55.238495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:55.238986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:10:55.239630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:55.241116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a06e6a6b-d671-4034-8df3-36e930ad3640]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:55.242552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80b377fb-49e0-4426-a455-e3314c6c5ca9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:55.244457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9aceb6e2-278e-4c13-94b6-4c3af58147a3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:55.280492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78415942-ec92-4f84-aed4-447793869753]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 515119, 'reachable_time': 43788, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 178336, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:55.285970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:10:55.286860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d459ac7d-835b-4d82-aef0-85685d4f667f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:55.291130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-DeleteServersTestJSON-74730730-router", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100" Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:36Z", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:39Z", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7a:1f:c1", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4082, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:37Z", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:41Z", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd38837b-22e9-45db-8d27-924c76e385db", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a9a659de-f6d1-42d7-9d38-bd056dfcb759", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3f:9f:bb", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4086, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:39Z", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:53Z", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:10:55.296395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:10:55.309144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 849d8323-a39b-485f-b6ca-0b561268cc22. Wait time elapsed: 14.929 Jul 01 10:10:55.309144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 849d8323-a39b-485f-b6ca-0b561268cc22. Wait time elapsed: 14.931 Jul 01 10:10:55.309144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:10:57.888051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.589s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:10:57.888605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 893e74e1-66a6-40a4-91fa-1a8006c88697 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:10:57.889426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:10:57.889996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:57.890490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:57.892766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32780878-9430-46c8-8333-e53841a670ee]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:57.917529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6bbc0889-2435-4e7f-bddc-bd4cf1dd6d64]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:57.942223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4556fbcc-edd0-4b93-885a-2b0738188ddd]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:57.996341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[873e0b9c-5adb-42f4-b630-426e45cf6e67]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 178391, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.030948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfa44b3f-d257-46cd-b896-8a7f9609ec33]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.054600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ddee01f-0a7e-43ef-aa92-5a23cb410d0e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.076664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c4b59c8-7046-44ca-b570-b37e57af3ba4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.098440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dcd922f5-3352-4330-bedf-c55dd2feefa5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.099456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:58.099741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:58.100012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:10:58.100296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd38837b-22e9-45db-8d27-924c76e385db) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:10:58.117442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:10:58.117728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[329da808-e1f3-4d5c-9e87-d86b37d9bdff]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.121204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8cd5d07-18a0-49ba-83ff-6787a65b427e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.122364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd38837b-22, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:58.158439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd38837b-22, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:58.159114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:58.159540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:58.160105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:58.160650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd38837b-22, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd38837b-22e9-45db-8d27-924c76e385db', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3f:9f:bb'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:58.216268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8943c70a-deba-4552-97f9-863a8d127c12]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.253861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[169e4742-d3af-488b-9540-3bfb4e55b6fb]: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:10:58.253861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:10:58.253861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:10:58.253861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:10:58.253861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:10:58.253861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:10:58.253861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:10:58.253861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:10:58.253861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:10:58.253861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:10:58.253861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:10:58.253861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:10:58.253861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:58.254926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[169e4742-d3af-488b-9540-3bfb4e55b6fb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:58.255109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:58.257161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:58.260484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3835588a-39b4-4c20-bb9b-e69c6bd166f6]: (4, ['qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.260484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:58.260484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:58.260484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:10:58.260484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:58.261925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63d8df07-d6a8-4d23-af42-19ed9f241737]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.263229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e79895b3-8694-43e7-93b2-3174409e5a48]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.264420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8b28c33-2663-4a7b-816e-c02c099136f5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.291300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[938c8dfa-c3fe-41c8-a48c-9ea69ab3864e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 515426, 'reachable_time': 20164, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 178411, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.298776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:10:58.299241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d67870e6-3d9a-4923-a326-7d8b916f88d2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:58.300392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-DeleteServersTestJSON-74730730-router", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100" Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:36Z", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:39Z", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7a:1f:c1", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4082, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:37Z", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:41Z", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd38837b-22e9-45db-8d27-924c76e385db", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a9a659de-f6d1-42d7-9d38-bd056dfcb759", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3f:9f:bb", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4086, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:39Z", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:56Z", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:10:58.302979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 07156416-3594-486a-b1a0-646493cd7704. Wait time elapsed: 19.699 Jul 01 10:10:58.322221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 07156416-3594-486a-b1a0-646493cd7704. Wait time elapsed: 19.699 Jul 01 10:10:58.322221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:10:59.045939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.743s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:10:59.046932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 893e74e1-66a6-40a4-91fa-1a8006c88697 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:10:59.049566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:10:59.050386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:59.051489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:59.053296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f5427d8-48ad-4dfe-8041-e864ee1c9944]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.074045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[227d2bae-4aee-44b8-a9d2-2d055e42df7e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.101385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efa2a84c-39b3-49dd-a831-b06d423f1cd3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.134831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ba97195-58aa-4333-aefe-6f0ee871d1bd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 178418, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.157051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a82ced9-1b26-43c1-adfe-e727be6a6e5e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.184221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ee31fc5-2b4c-4f13-b7a5-ed678f211f73]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.209531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8922b3f3-5c69-4318-b937-f71f2823f35b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.245680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2bc6384d-2b9a-4cdd-b787-828ea90e3b7e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.246250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:10:59.246550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:10:59.246838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:10:59.247121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd38837b-22e9-45db-8d27-924c76e385db) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:10:59.263713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:10:59.263998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6ad3eab-7422-4136-ae7e-88a01e0f70df]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.268203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4fa1425-ff80-4519-b604-bf6a6468feb7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.269147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd38837b-22, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:59.302129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd38837b-22, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:59.302813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:59.303211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:59.303634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:59.304020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd38837b-22, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd38837b-22e9-45db-8d27-924c76e385db', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3f:9f:bb'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:10:59.346638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea52d93d-8de9-4461-80c0-e8e8a2be9825]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.377792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[059a4bc7-dc38-42f0-a874-8bb15bcb5ea8]: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:10:59.377792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:10:59.377792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:10:59.377792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:10:59.377792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:10:59.377792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:10:59.377792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:10:59.377792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:10:59.377792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:10:59.377792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:10:59.377792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:10:59.377792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:10:59.377792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:59.378901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[059a4bc7-dc38-42f0-a874-8bb15bcb5ea8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:59.379012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:59.381242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:59.384163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[440d701e-a83f-4aa5-a6f1-0f909ab0cc1b]: (4, ['qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.384163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:59.384163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:59.384163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:10:59.384556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:10:59.385445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3bf76cd-ee29-4723-bcd3-f33e964bb8a0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.386365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ea42a9d-7cf5-4ba1-a2cd-eeb3ad6415c5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.387321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db1bbf0c-cff3-41ae-8b28-6d33236ccb56]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.408649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20bf18ce-0f1a-4d2b-a7cb-440e5b96af37]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 515539, 'reachable_time': 20300, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 178437, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.410918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:10:59.411397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c4a6331-253f-465d-aaeb-cb423fc79f69]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:10:59.412674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-DeleteServersTestJSON-74730730-router", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100" Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:36Z", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:39Z", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7a:1f:c1", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4082, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:37Z", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:41Z", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd38837b-22e9-45db-8d27-924c76e385db", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a9a659de-f6d1-42d7-9d38-bd056dfcb759", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3f:9f:bb", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4086, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:39Z", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:57Z", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:10:59.416480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:10:59.427430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 849d8323-a39b-485f-b6ca-0b561268cc22. Wait time elapsed: 19.049 Jul 01 10:10:59.427430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 849d8323-a39b-485f-b6ca-0b561268cc22. Wait time elapsed: 19.049 Jul 01 10:10:59.427430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:11:01.980335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.563s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:11:01.980981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 893e74e1-66a6-40a4-91fa-1a8006c88697 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:11:01.981623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:01.981948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:01.982228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:01.983389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41ae78f7-eb88-440b-971b-d52f5b663736]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.000555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed2ebd4b-1896-48e9-83b5-1e950f865bb0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.021912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01ce42e8-b893-4a73-8c17-249da5bb417b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.052684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cbb439b-fd3f-439c-9c37-c426d172d70c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 178486, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.075124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd425d68-a5d7-43a2-9931-e44192793587]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.095638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[082ce2f5-4f61-4e7e-b9e1-fa107eeea680]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.115596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[994c4c0a-a3a8-49c3-bb27-101302768a3e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.136514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c783285-1c28-42cb-9810-b87971499801]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.137062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:02.137360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:02.137626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:11:02.137933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd38837b-22e9-45db-8d27-924c76e385db) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:11:02.156068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:11:02.156364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1593b85e-7a7a-40be-97e2-e19e180c4f8f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.159405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d258af79-b771-4b54-a872-fd1870f7b0e7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.160570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd38837b-22, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:02.197155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd38837b-22, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:02.197628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:02.198046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:02.198605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:02.199143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd38837b-22, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd38837b-22e9-45db-8d27-924c76e385db', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3f:9f:bb'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:02.241383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66b6cd29-816a-44b9-8920-afc06ff21e0b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.283586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9e913788-f18c-4b30-a288-00702707d490]: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:11:02.283586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:11:02.283586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:11:02.283586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:11:02.283586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:11:02.283586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:11:02.283586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:11:02.283586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:11:02.283586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:11:02.283586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:11:02.283586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:11:02.283586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:11:02.283586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:02.284706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e913788-f18c-4b30-a288-00702707d490]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:02.284850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:02.286647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:02.288994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99961408-ae0c-460d-b5eb-a207a5534e4b]: (4, ['qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.289651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:02.290107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:02.290369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:11:02.290793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:02.291917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e586d7cd-e0bf-49e8-a701-09011003b9d0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.292850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b58bb66-25da-4b90-845a-708cd535c4c4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.293720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[641c6a3b-501c-4abb-8474-a6a523da7046]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.326980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2501771-a3c0-45a0-bc35-3e00ef956191]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 515831, 'reachable_time': 28228, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 178504, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02e90705-b07e-493c-a119-86c6e18c183c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-DeleteServersTestJSON-74730730-router", Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100" Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:11:02.332507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:36Z", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:39Z", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7a:1f:c1", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4082, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:37Z", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:41Z", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd38837b-22e9-45db-8d27-924c76e385db", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a9a659de-f6d1-42d7-9d38-bd056dfcb759", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3f:9f:bb", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4086, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:39Z", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:00Z", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 07156416-3594-486a-b1a0-646493cd7704. Wait time elapsed: 23.730 Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 07156416-3594-486a-b1a0-646493cd7704. Wait time elapsed: 23.731 Jul 01 10:11:02.339263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:11:03.123295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.790s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:11:03.123920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 893e74e1-66a6-40a4-91fa-1a8006c88697 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:11:03.125192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:03.125582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:03.125933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:03.127142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acb2b0da-269a-425e-920a-1963d64e6e71]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.139985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[073c4e78-77da-45e5-8542-438d8f5908ee]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.162390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5f0e160-c7d4-40ec-a1c6-1c310816a86e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.192104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[732566ca-a3ba-45bf-ad2a-9a20f9b46582]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 178508, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.213141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11d7d805-7f35-46c3-971f-1f995f50af8e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.233639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08e7b90a-657b-4949-9afc-6c17760fb607]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.253706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69969ec9-650c-4db9-aadd-7e4b3810d542]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.274517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c47ca41-abba-4a61-bcd5-056e8c78c01e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.275255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:03.275582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:03.275982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:11:03.276294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd38837b-22e9-45db-8d27-924c76e385db) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:11:03.292231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:11:03.292531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[177847ad-2cef-4c3f-a282-194a545aee0b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.295185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a7c5305-91d9-4184-bb26-fe4ca48dc34b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.296186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd38837b-22, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:03.327774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd38837b-22, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:03.328889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:03.329515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:03.330410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:03.330937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd38837b-22, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd38837b-22e9-45db-8d27-924c76e385db', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3f:9f:bb'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:03.367416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6248442-bc96-47f4-b21e-b31360171b53]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.400666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[df0939cb-0e0f-428f-baf5-31ecd4f266e5]: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:11:03.400666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:11:03.400666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:11:03.400666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:11:03.400666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:11:03.400666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:11:03.400666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:11:03.400666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:11:03.400666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:11:03.400666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:11:03.400666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:11:03.400666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:11:03.400666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:03.401705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df0939cb-0e0f-428f-baf5-31ecd4f266e5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:03.401959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:03.403795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:03.406037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a37b839d-da41-4e6c-a88d-bb497febecf1]: (4, ['qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.406037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:03.406037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:03.406037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:11:03.406369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:03.407261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12554cb4-9a3c-416d-9fc7-ae4c28cfe931]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.408406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bdbde61e-c939-4401-bca2-fda498b1973d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.409401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[047cc230-ff04-4cdd-b88a-ce18383f40f8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.429938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a11fa4a5-0826-42e5-9335-5ff9c5927c89]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 515945, 'reachable_time': 27112, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 178526, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0054f20-69d5-4915-b905-4d4502afc082]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-DeleteServersTestJSON-74730730-router", Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100" Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:11:03.435230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:36Z", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:39Z", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7a:1f:c1", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4082, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:37Z", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:41Z", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd38837b-22e9-45db-8d27-924c76e385db", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a9a659de-f6d1-42d7-9d38-bd056dfcb759", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3f:9f:bb", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4086, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:39Z", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:02Z", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3 Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 849d8323-a39b-485f-b6ca-0b561268cc22. Wait time elapsed: 23.068 Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 2, update_id 849d8323-a39b-485f-b6ca-0b561268cc22. Wait time elapsed: 23.068 Jul 01 10:11:03.440925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:11:05.970749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.535s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:11:05.971373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 893e74e1-66a6-40a4-91fa-1a8006c88697 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:11:05.972068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:05.972389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:05.973644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:05.973991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c32919eb-8781-433e-851a-70ec9546ba30]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:05.987062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad68e6e7-fcc1-4706-ba15-a5a52d21cc72]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.008949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd025f20-9339-4668-9d65-f5a398a94df5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.045863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[822f6a72-1ca3-48c2-9264-cd922810459d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 178576, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.066465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53627a2e-c1a2-4462-bc44-8b2ecd788807]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.089380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f457050f-c2db-486a-b6bb-293de3aae55d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.111836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[528bc56c-1c97-4b4c-9e60-15df7598535d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.136389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09227445-a8c6-49ef-8513-044bd6e840b1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.137351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:06.137698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:06.138031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:11:06.138375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd38837b-22e9-45db-8d27-924c76e385db) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:11:06.164222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:11:06.164814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c151846f-a887-403b-b595-a393cd6cc1d3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.177539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a391cbcf-9bb7-4158-9755-8b3419f7354d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.178623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd38837b-22, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:06.217785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd38837b-22, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:06.219258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:06.220350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:06.220956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd38837b-22, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:06.221624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd38837b-22, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd38837b-22e9-45db-8d27-924c76e385db', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3f:9f:bb'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:06.267168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7317696-42e8-44d4-ba43-1edee6041202]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e35ef845-3c75-4580-bfea-cca074d1b895]: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:11:06.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:11:06.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:11:06.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:11:06.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:11:06.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:11:06.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:11:06.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:11:06.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:11:06.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:11:06.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:11:06.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:11:06.299902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:06.300847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e35ef845-3c75-4580-bfea-cca074d1b895]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:06.301669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:06.303167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:06.305762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eaa11d6c-99d5-4e3c-aed6-d6e0a7282c50]: (4, ['qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.305762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:06.305762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:06.305762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:11:06.306187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:06.307515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73a35612-1971-4c97-9966-0391b8e57dba]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.308488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a08d206c-c41a-49eb-893b-51719dbd1289]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.309454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a55f63cc-ccd2-468c-8b86-661b83be46b6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.328850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2aec11b8-74b1-42d3-9de9-7a989ebd776c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 516230, 'reachable_time': 36168, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 178594, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.335007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:11:06.335259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e3f71a2-0918-470e-bf09-bcef59a6de22]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 893e74e1-66a6-40a4-91fa-1a8006c88697: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd38837b-22 not found in namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697. Jul 01 10:11:06.336294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-DeleteServersTestJSON-74730730-router", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100" Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:36Z", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:39Z", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "17c5b8ea-ceb1-467c-a844-03b50c15b9cd", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:7a:1f:c1", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4082, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.100", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:37Z", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:10:41Z", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd38837b-22e9-45db-8d27-924c76e385db", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a9a659de-f6d1-42d7-9d38-bd056dfcb759", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3f:9f:bb", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "893e74e1-66a6-40a4-91fa-1a8006c88697", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4086, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:10:39Z", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:03Z", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "38aa9fbdf71f4e2994d1180ba9361183", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48466ec-8087-45a4-9905-e16b583de7d9", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:11:06.339677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3 Jul 01 10:11:15.119132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:15.119743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:15.119833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:15.120025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:34.479623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-cdc661a4-f8b9-4606-b919-6b133166ec86 admin admin] Got routers updated notification :['893e74e1-66a6-40a4-91fa-1a8006c88697'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:11:34.480770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 1, update_id 11e87bc2-2caa-44a7-b75d-b4b1d1367972. Wait time elapsed: 0.001 Jul 01 10:11:34.481138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 3, priority 1, update_id 11e87bc2-2caa-44a7-b75d-b4b1d1367972. Wait time elapsed: 0.001 Jul 01 10:11:34.481497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:11:35.062759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.581s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:11:35.063114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 893e74e1-66a6-40a4-91fa-1a8006c88697 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:11:35.064743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:35.065135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:35.065489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:35.066832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67123358-3308-41df-93b7-d5039b799ca3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.088568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d4430c8-244a-4d4f-b1e2-311c824545fd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.110445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[928bf4d7-9033-4814-b4a1-4c688d2530eb]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.145153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bbc4418-19c1-45f4-bc8f-7dd1cb30e034]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 179012, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.169744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec75f0e1-3440-476b-8c38-697f8f62f884]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.193345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cdffa4b-99b4-457c-a3df-9f0e1bba0847]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.208389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47564cf7-4f0f-4280-8661-85dcbe5675b4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.230394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:11:35.236155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9fef355b-9e47-44d6-9386-1b5b98363ea4]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.236714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:35.237192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:35.237464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:11:35.265843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58c03231-c127-40d6-91a7-b15ca6264ee5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 519140, 'reachable_time': 27744, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179018, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.267282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:35.268120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:35.268878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:11:35.269607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:11:35.310510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca71df45-4b17-4492-88d3-c1cc83f186f7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 519140, 'reachable_time': 27744, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179020, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.311474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:11:35.311955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:11:35.330267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8275dda9-5cd6-448c-b2c1-af733489dc77]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.432347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b9a27a62-3cbe-43b5-b8e4-10a2874479bd admin admin] Got router deleted notification for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:11:35.550671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0e618c3-5dd4-4ba4-aa12-0cf5a72bc84d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.599541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e6307a9-e207-40b7-a9fb-633d52279d7c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.782409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d9a3ec1-ea16-464a-b914-cdbce40b6ccd]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.782969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:11:35.783348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:11:35.783736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:11:35.784126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" released by "process_external" :: held 0.514s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:11:35.784801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:11:35.785203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:11:35.785642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:11:35.786155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:11:35.801648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cf505b9-80cc-45ea-bfde-6921b53be28a]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.844940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66ef7dc8-7224-4c33-acd7-4f9c55b55ca6]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.860127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50e40356-0d83-489d-87be-b63d373a17fe]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.911084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55348ccf-9d02-4654-be0f-d35878cd1758]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.912538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:11:35.914259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:11:35.914670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:11:35.915769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" released by "process_address_scope" :: held 0.130s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:11:35.916313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:35.917022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:11:35.917476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" acquired by "apply_metadata_nat_rules" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:11:35.918800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:11:35.919465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:11:35.937746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[763260d1-0b51-41cf-8ee4-31c93f9bbfb1]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:35.982741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bc887fd-ab64-4449-a15e-ffa3bd0c12dc]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:36.000773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a478ff4-d94b-4115-8045-12b485420cb3]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:11:35 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 10:11:35 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:36.058300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[461e1554-f259-40e4-a0ec-efa105bcf1ad]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:36.059304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:11:36.059736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:11:36.060119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:11:36.060520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697" released by "apply_metadata_nat_rules" :: held 0.143s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:11:36.061116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-893e74e1-66a6-40a4-91fa-1a8006c88697 Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.haproxy Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 893e74e1-66a6-40a4-91fa-1a8006c88697 Jul 01 10:11:36.061746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 10:11:36.064081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/893e74e1-66a6-40a4-91fa-1a8006c88697.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 10:11:36.085288 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-893e74e1-66a6-40a4-91fa-1a8006c88697[179078]: Proxy listener started. Jul 01 10:11:36.088601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 893e74e1-66a6-40a4-91fa-1a8006c88697 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 10:11:36.088870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, update_id 11e87bc2-2caa-44a7-b75d-b4b1d1367972. Time elapsed: 1.608 Jul 01 10:11:36.089147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 893e74e1-66a6-40a4-91fa-1a8006c88697, action 1, priority 1, update_id 3219c913-6e4b-43dc-bd1a-172e9870d4ef. Wait time elapsed: 0.657 Jul 01 10:11:36.089391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 893e74e1-66a6-40a4-91fa-1a8006c88697, action 1, priority 1, update_id 3219c913-6e4b-43dc-bd1a-172e9870d4ef. Wait time elapsed: 0.657 Jul 01 10:11:36.089681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 893e74e1-66a6-40a4-91fa-1a8006c88697 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 10:11:36.089991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:36.109598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d543cdb-bb55-4dd1-9a82-b6a1cd97aa02]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:36.111183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d388f1c0-11a2-40d3-914b-13dd79622ff7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:36.112342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5d11ab6-4a8b-4d6b-bea2-369d48b5d2e1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:36.112831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 10:11:36.113627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e847c80-db2e-4c5e-97d3-2fdbb50ad523]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:36.133146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd117345-9c55-48a1-898e-54598ebabdf9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 519140, 'reachable_time': 27744, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179084, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:36.134245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:36.134870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:36.154447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44fecd5c-ec37-44e6-8af1-96dc31563922]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 519140, 'reachable_time': 27744, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179085, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:36.155472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 10:11:36.155951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:36.156573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/893e74e1-66a6-40a4-91fa-1a8006c88697.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:36.156827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:11:36.157574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 893e74e1-66a6-40a4-91fa-1a8006c88697 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 10:11:36.158658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6f56445-3913-4183-a87c-42119d7f6cdf]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:36.181762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29810184-ddd1-4cdd-a63e-9bf01f49191b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 519140, 'reachable_time': 27744, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179086, 'error': None, 'target': 'qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:36.192560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-893e74e1-66a6-40a4-91fa-1a8006c88697 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:11:36.192870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[015f8e28-5a1a-43d5-88b2-6560a507a800]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:36.193329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 893e74e1-66a6-40a4-91fa-1a8006c88697 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 10:11:36.193673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (893e74e1-66a6-40a4-91fa-1a8006c88697), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:36.195776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:36.195776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:36.196045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:36.196300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:36.196554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 893e74e1-66a6-40a4-91fa-1a8006c88697, update_id 3219c913-6e4b-43dc-bd1a-172e9870d4ef. Time elapsed: 0.107 Jul 01 10:11:39.797794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-dae0d011-b7a0-47a5-9495-7d9e405b9ed1 admin admin] Got routers updated notification :['fd3eae0a-8937-45ea-8cb4-ab69b5ec790b'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:11:39.798476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 1, update_id 6b447daf-1f0a-4859-816d-d4ff632e4ef2. Wait time elapsed: 0.000 Jul 01 10:11:39.798574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 1, update_id 6b447daf-1f0a-4859-816d-d4ff632e4ef2. Wait time elapsed: 0.000 Jul 01 10:11:39.798862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:11:41.563157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-d70707e1-9c91-42bb-8727-651f7b8869ee admin admin] Got routers updated notification :['fd3eae0a-8937-45ea-8cb4-ab69b5ec790b'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:11:43.347127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.548s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:11:43.348174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:11:43.349227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (fd3eae0a-8937-45ea-8cb4-ab69b5ec790b), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:43.349551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:43.349846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:43.356360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a230c9dc-67f2-4e11-9ccf-0edc14d4dc00]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.377487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72dbb472-92f6-4e90-9e9e-cb4b900763df]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.408135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[010aece8-2393-443f-8e19-64abfab937c4]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.457484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b8f7fc8-a45b-4efc-b825-61a4974fe8e7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 179175, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.480148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[147871a5-f7d4-4939-a344-82461ea3921e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.511297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[424da61b-eaee-4000-8604-847a7af079bf]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.535133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f660b537-8c5d-4d75-8673-b608fdbbf241]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.583916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[007f64ae-9e47-4b57-9ea4-c6551baef4c3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.584750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:43.585196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:43.586390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:11:43.586828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dcd9d026-2e1c-4537-991d-e3e79d3864ed) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:11:43.606223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:11:43.606844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0cefb7c9-88b7-4255-aae2-5372203a55ac]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.609549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb212628-6ea9-4ff3-8b6a-206737116b7e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.610617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dcd9d026-2e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:43.611106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:11:43.611990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dcd9d026-2e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:43.612731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:43.613190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:43.613731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:43.614343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dcd9d026-2e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dcd9d026-2e1c-4537-991d-e3e79d3864ed', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4a:66:f4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:43.663719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb681f9d-a3d2-41f3-8d9f-0d13d4385431]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.703634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[667a3aa9-eb6d-48c4-9ef4-b639325ceb28]: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:11:43.703634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:11:43.703634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:11:43.703634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:11:43.703634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:11:43.703634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:11:43.703634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:11:43.703634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:11:43.703634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:11:43.703634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:11:43.703634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:11:43.703634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:11:43.703634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:43.704596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[667a3aa9-eb6d-48c4-9ef4-b639325ceb28]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:43.705126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:43.707035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:43.709821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ecca524-df4e-4229-9355-8ff669ce2a0e]: (4, ['qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.710650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:43.711040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:43.711331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:11:43.711718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:43.712825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c9ec978-2f11-49bf-81a0-ec8d9a30f740]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.713733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9738cf3-fade-44ce-8144-e13ca331feb3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.714671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a634762-ed01-4959-8660-d2f4b4b7c733]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.735868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5dac3c2d-ebbb-43b6-95dd-53dcfa01be09]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 519972, 'reachable_time': 44836, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179193, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.742760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:11:43.743146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d718082-178e-4964-9319-9b4abae5c8f8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:43.744447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerDiskConfigTestJSON-1194626940-router", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171" Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:40Z", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:30:21:9b", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4141, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:42Z", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dcd9d026-2e1c-4537-991d-e3e79d3864ed", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "734ab8c1-e6b3-4c59-a80b-0d64c2042a5d", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4a:66:f4", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4145, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:40Z", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:40Z", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 2, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 1, update_id 0c643ec4-5675-470c-b48c-12af8e5d7119. Wait time elapsed: 2.181 Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 1, update_id 0c643ec4-5675-470c-b48c-12af8e5d7119. Wait time elapsed: 2.182 Jul 01 10:11:43.747103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:11:45.845648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.099s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:11:45.846292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:11:45.847103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (fd3eae0a-8937-45ea-8cb4-ab69b5ec790b), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:45.847480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:45.847766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:45.848950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f79b9d00-5c21-4d7d-9dec-70201d6af0d2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:45.888620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36b53207-9199-48f4-9f1f-ee416853ac51]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:45.916459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78d19e81-e786-475b-85ff-d0e3898a46b4]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:45.945982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d98fae1b-d5e2-49ce-a4d3-3c525dbad436]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 179242, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:45.967588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[405443f7-3f14-4708-a73f-9d554b2182ff]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:45.990554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a88264b-60f0-49d0-a752-c6ff264489f1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:46.012268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1e50488-c9ef-49b4-b646-d3bf8204d138]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:46.033241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6535cacf-edb7-41c9-95d6-d2745b880042]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:46.033853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:46.034135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:46.034407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:11:46.034691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dcd9d026-2e1c-4537-991d-e3e79d3864ed) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:11:46.051964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:11:46.052504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f48db1ef-c00e-4080-a6a5-cc9108dda7d1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:46.056484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07df38fa-7736-43f3-8824-d4f79be56976]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:46.057892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dcd9d026-2e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:46.098262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dcd9d026-2e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:46.099507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:46.102128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:46.103803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:46.104771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dcd9d026-2e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dcd9d026-2e1c-4537-991d-e3e79d3864ed', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4a:66:f4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:46.164659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c526bdfa-daa8-4222-bf7a-21b8c14660c7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:46.199511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e4339910-4e9e-4b7a-99f1-1e5dd3466711]: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:11:46.199511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:11:46.199511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:11:46.199511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:11:46.199511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:11:46.199511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:11:46.199511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:11:46.199511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:11:46.199511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:11:46.199511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:11:46.199511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:11:46.199511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:11:46.199511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:46.200700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4339910-4e9e-4b7a-99f1-1e5dd3466711]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:46.201235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:46.203607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:46.206144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9eaaca0a-0803-49bd-ad76-994216a8d92f]: (4, ['qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:46.206144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:46.206144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:46.206144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:11:46.206144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:46.206790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[123d2f12-b2a9-4d0d-94f8-ee1f40db70d2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:46.207859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[219cb5be-aea5-4681-a400-6b82d518031b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:46.208930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[754198b4-6ecc-4b86-9860-b664f7972c94]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:46.227689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81fed1cb-105c-4cad-8d35-fb5597eea542]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 520220, 'reachable_time': 32468, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179262, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:46.233714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:11:46.234458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57c9c8d3-cfa7-4eaa-aa21-b235d01b3191]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:46.235644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerDiskConfigTestJSON-1194626940-router", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171" Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:40Z", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:30:21:9b", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4141, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:42Z", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dcd9d026-2e1c-4537-991d-e3e79d3864ed", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "734ab8c1-e6b3-4c59-a80b-0d64c2042a5d", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4a:66:f4", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4145, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:40Z", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:45Z", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:11:46.238794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 6b447daf-1f0a-4859-816d-d4ff632e4ef2. Wait time elapsed: 6.439 Jul 01 10:11:46.253487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 6b447daf-1f0a-4859-816d-d4ff632e4ef2. Wait time elapsed: 6.439 Jul 01 10:11:46.253487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:11:48.328460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.091s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:11:48.329074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:11:48.329847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (fd3eae0a-8937-45ea-8cb4-ab69b5ec790b), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:48.330346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:48.330634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:48.332361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38912e4d-02fa-4415-adb7-987a28536c41]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.349569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[644eb5db-630b-4817-a5c3-8113ce5086ad]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.372736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d99146b3-9a97-44d6-8495-07671eceed6e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.405498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e2422a6-4b36-42af-ab3f-59697b48758e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 179269, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.433152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb21ded1-876e-4150-bc00-9955190de3a3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.455586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[afbe2668-a82b-43d5-a4bd-42aabc5e73f8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.477504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59607f1d-63be-457c-8def-8826bae385ff]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.499176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6eb6f87-370e-4b28-9d5e-6f9351c4c474]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.499602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:48.499880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:48.500148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:11:48.500430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dcd9d026-2e1c-4537-991d-e3e79d3864ed) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:11:48.517281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:11:48.518107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e6d82a0-9a17-4eca-9bb6-f8ca44646421]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.521102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4f0597e-cb19-4e69-af46-ec8ad5b62182]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.522274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dcd9d026-2e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:48.556039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dcd9d026-2e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:48.557125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:48.558977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:48.559908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:48.560529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dcd9d026-2e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dcd9d026-2e1c-4537-991d-e3e79d3864ed', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4a:66:f4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:48.622431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd657701-b249-43b2-a37b-9c5ec34a150b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.666350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[46d1c30e-5c9f-4f9a-80a8-9cbf3061ec1c]: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:11:48.666350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:11:48.666350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:11:48.666350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:11:48.666350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:11:48.666350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:11:48.666350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:11:48.666350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:11:48.666350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:11:48.666350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:11:48.666350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:11:48.666350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:11:48.666350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:48.667306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46d1c30e-5c9f-4f9a-80a8-9cbf3061ec1c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:48.668172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:48.669778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:48.671898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de27a4ef-f656-4e15-b4b5-d261455e90a3]: (4, ['qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.671898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:48.671898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:48.671898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:11:48.671898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:48.674777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee4f4ccc-5204-4902-9344-cec92d2ca00b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.676340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81dad412-fe81-4675-a804-9ed76de84621]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.678118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f4e9c08-50c1-4596-8ab7-9ecc2f6d10f0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.704338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f814101f-1867-4d3e-9216-3fc91043a9fb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 520466, 'reachable_time': 22316, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179288, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.714065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:11:48.714459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57bcb07c-0dcb-4863-a125-6622d01e12ef]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:48.715614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerDiskConfigTestJSON-1194626940-router", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171" Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:40Z", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:30:21:9b", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4141, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:42Z", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dcd9d026-2e1c-4537-991d-e3e79d3864ed", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "734ab8c1-e6b3-4c59-a80b-0d64c2042a5d", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4a:66:f4", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4145, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:40Z", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:45Z", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:11:48.719032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:11:48.731921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 0c643ec4-5675-470c-b48c-12af8e5d7119. Wait time elapsed: 7.156 Jul 01 10:11:48.731921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 0c643ec4-5675-470c-b48c-12af8e5d7119. Wait time elapsed: 7.157 Jul 01 10:11:48.731921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:11:49.465132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.739s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:11:49.465764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:11:49.466330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (fd3eae0a-8937-45ea-8cb4-ab69b5ec790b), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:49.467161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:49.467486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:49.468612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[585df3cb-b289-4c17-8910-3d08563c5418]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.479408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[273fa548-6149-4b17-8f6a-5bf780d68be6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.506024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e78f6e9-6e9c-4d6a-9213-b650a3411a3d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.538817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbc07bbe-3999-4192-ba70-9879c0c36943]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 179291, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.559804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31750e51-7c78-4eb8-b257-a8c20852de74]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.580167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9c5b867-7a09-4891-84d9-134e4d194bf4]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.604955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de17a5b3-1042-430a-bec7-fcffc146d648]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.628250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31941bf4-2ad8-4684-9473-a641bbd8f4cf]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.628797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:49.629099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:49.629365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:11:49.629644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dcd9d026-2e1c-4537-991d-e3e79d3864ed) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:11:49.646456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:11:49.646753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97682e92-7dc1-49ab-b8b8-77e99865d68f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.650281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95dada3f-c313-4056-b438-b3927116bded]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.651316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dcd9d026-2e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:49.701531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dcd9d026-2e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:49.702521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:49.702912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:49.703297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:49.703708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dcd9d026-2e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dcd9d026-2e1c-4537-991d-e3e79d3864ed', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4a:66:f4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:49.758121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df480f65-fa00-4762-b23e-4454a2a2b71c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.802718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[095be5e3-91de-4555-bc7a-68f486ac38e3]: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:11:49.802718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:11:49.802718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:11:49.802718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:11:49.802718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:11:49.802718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:11:49.802718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:11:49.802718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:11:49.802718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:11:49.802718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:11:49.802718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:11:49.802718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:11:49.802718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[095be5e3-91de-4555-bc7a-68f486ac38e3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:49.805746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:49.809368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:49.809368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:49.809368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:49.809368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:49.809368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:49.809368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:49.809368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:49.809368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:49.809368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:49.809368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:49.809368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:49.809368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:49.809368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9d0dce9-de23-44f2-80be-f34c602d96ee]: (4, ['qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.811033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:49.811033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:49.811033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:11:49.811033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:49.811844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27e0a0b7-49ac-4b54-b850-fd1b8044b86c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.813092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[081a77ab-caf7-4848-bdb1-864300f0dfa8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.813875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69b1a8ac-2e4c-49e0-b072-29beba92d63e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.835013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[389e60f2-2671-45d9-adc8-9cbb01c393d8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 520580, 'reachable_time': 31492, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179310, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.841754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:11:49.842184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab7c32ed-fdb2-40c3-a267-41ed8d274798]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:49.843285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerDiskConfigTestJSON-1194626940-router", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171" Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:40Z", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:30:21:9b", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4141, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:42Z", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dcd9d026-2e1c-4537-991d-e3e79d3864ed", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "734ab8c1-e6b3-4c59-a80b-0d64c2042a5d", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4a:66:f4", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4145, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:40Z", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:47Z", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:11:49.846259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 6b447daf-1f0a-4859-816d-d4ff632e4ef2. Wait time elapsed: 10.046 Jul 01 10:11:49.864335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 6b447daf-1f0a-4859-816d-d4ff632e4ef2. Wait time elapsed: 10.047 Jul 01 10:11:49.864335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:11:52.639933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.795s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:11:52.640535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:11:52.641112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (fd3eae0a-8937-45ea-8cb4-ab69b5ec790b), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:52.641622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:52.641901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:52.643027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d739f34-db7f-4f0e-ad25-1d2ac3d4ac31]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.657367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c48ba45-beed-47ba-98e1-4974f87d87d6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.680467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b98e52c-f408-4536-8c5c-94c4801b3d35]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.716842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52835dbe-7cff-4bf3-bc94-324fe2f8b6a3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 179362, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.737663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c12d4d8b-f917-494d-abbb-8cd49aca509d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.758052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3488d6a-2784-412d-a954-eb4be33a15fd]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.778137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11ad2cc9-f6a1-48b5-aa4b-e6914dfeb206]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.798701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd1a9a50-5629-4239-9b94-ff3fd5c4a42a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:52.799536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:52.799852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:11:52.800131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dcd9d026-2e1c-4537-991d-e3e79d3864ed) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:11:52.817217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:11:52.817553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6601234e-7fd3-44db-a11e-6d624da4659e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.821092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da07eb6d-34ee-4096-96e1-8fea3cd73e77]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.822180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dcd9d026-2e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:52.863718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dcd9d026-2e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:52.864394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:52.864785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:52.865169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:52.865576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dcd9d026-2e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dcd9d026-2e1c-4537-991d-e3e79d3864ed', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4a:66:f4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:52.902932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[204f5108-8bb1-4a01-a3d8-3da641ef54fb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.940142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ca42bf7d-504f-4f10-a21e-d4e064694542]: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:11:52.940142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:11:52.940142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:11:52.940142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:11:52.940142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:11:52.940142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:11:52.940142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:11:52.940142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:11:52.940142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:11:52.940142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:11:52.940142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:11:52.940142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:11:52.940142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:52.941288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca42bf7d-504f-4f10-a21e-d4e064694542]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:52.941379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:52.943200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:52.946447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbb0d542-b814-49d4-bbe3-8bc5337836e6]: (4, ['qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.946447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:52.946447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:52.946447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:11:52.946447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:52.946447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[236f868d-cffc-4847-8c2d-743cacc5e8c6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.948144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6236ba0f-b2d4-409a-a4be-81f4555361e5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.949425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cfe37be4-9599-4987-bd62-f40f8a4bad0a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.973191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60aed8ca-1dd7-430c-a1f5-0ca07c2aaf10]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 520898, 'reachable_time': 25512, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179380, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.978806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:11:52.978806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45f2c5f6-71ac-4d19-bb91-f03f34956ef4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:52.979884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerDiskConfigTestJSON-1194626940-router", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171" Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:40Z", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:30:21:9b", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4141, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:42Z", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dcd9d026-2e1c-4537-991d-e3e79d3864ed", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "734ab8c1-e6b3-4c59-a80b-0d64c2042a5d", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4a:66:f4", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4145, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:40Z", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:51Z", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:11:52.982573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 0c643ec4-5675-470c-b48c-12af8e5d7119. Wait time elapsed: 11.417 Jul 01 10:11:52.993949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 0c643ec4-5675-470c-b48c-12af8e5d7119. Wait time elapsed: 11.417 Jul 01 10:11:52.993949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:11:53.776644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.795s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:11:53.777272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:11:53.778402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (fd3eae0a-8937-45ea-8cb4-ab69b5ec790b), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:53.778734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:53.779534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:53.780678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f5a468d-e419-47ee-b732-bc68c88992f0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:53.794498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3bfe0c3-7a87-4486-9cca-7a2b9694dc07]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:53.815109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7ea10cc-726a-4de7-bf1a-cd2d185fedfc]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:53.841673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[437ef676-25ab-4242-8e88-05beccd65567]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 179383, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:53.862894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80b242b6-315d-4d54-ba5f-8313bcc96c24]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:53.884880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b0dac53-f906-4fc4-972f-a4c4deefa2f1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:53.906203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3681b6b-dd7b-4d8a-a466-769f615eb409]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:53.928020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed7857e6-4b8e-4766-8ed6-5b8e56fdce7a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:53.928507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:53.928793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:53.929619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:11:53.930095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dcd9d026-2e1c-4537-991d-e3e79d3864ed) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:11:53.945744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:11:53.946123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cbf0086-afd9-43b7-bd48-47bf46d1dd8b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:53.948700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8aa1579-d5cd-473f-ab1b-77d3cba28bc8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:53.949652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dcd9d026-2e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:53.986506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dcd9d026-2e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:53.987194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:53.987623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:53.988025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:53.988496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dcd9d026-2e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dcd9d026-2e1c-4537-991d-e3e79d3864ed', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4a:66:f4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:54.045853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bc239b5-dcaf-45f4-bb29-7ad9fa0010f3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:54.077855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[38898ab0-d798-4087-8214-068df01279f4]: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:11:54.077855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:11:54.077855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:11:54.077855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:11:54.077855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:11:54.077855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:11:54.077855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:11:54.077855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:11:54.077855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:11:54.077855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:11:54.077855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:11:54.077855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:11:54.077855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:54.078961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38898ab0-d798-4087-8214-068df01279f4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:54.079232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:54.081102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:54.083518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11a4271d-dbc6-4493-a56a-25cd9a8d60eb]: (4, ['qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:54.083518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:54.083518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:54.083518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:11:54.083518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:54.084908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f14a519-4ef8-4931-a492-511e2a0575f9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:54.086531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[319146d7-9e51-4340-98f0-a4ceff8c8e5c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:54.087498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1048b7ba-7781-4401-86b7-0699a5966a8f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:54.107723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7609bbee-9a4b-49ce-80dd-43e4b7c3b6a3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 521010, 'reachable_time': 15308, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179402, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:54.114397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:11:54.114657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[423208cf-5662-4482-9bbf-abe54b027b31]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:54.115776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerDiskConfigTestJSON-1194626940-router", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171" Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:40Z", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:30:21:9b", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4141, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:42Z", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dcd9d026-2e1c-4537-991d-e3e79d3864ed", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "734ab8c1-e6b3-4c59-a80b-0d64c2042a5d", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4a:66:f4", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4145, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:40Z", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:53Z", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:11:54.118834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 6b447daf-1f0a-4859-816d-d4ff632e4ef2. Wait time elapsed: 14.319 Jul 01 10:11:54.138106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 6b447daf-1f0a-4859-816d-d4ff632e4ef2. Wait time elapsed: 14.319 Jul 01 10:11:54.138106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:11:57.212129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.088s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:11:57.212720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:11:57.213462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (fd3eae0a-8937-45ea-8cb4-ab69b5ec790b), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:57.214017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:57.214370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:57.215732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7a2a781-93db-4e95-844c-deb2a2803d9d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.238862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb59f9fe-c506-4762-9ebe-07115f8cce1b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.264313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1fd2ec8-4754-4088-b219-e77133aeabd4]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.297567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69e9cfc3-a087-4d80-84c7-a9604ecbea62]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 179455, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.324499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bc10cf4-04cf-4458-8de3-ec48be45c80b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.350901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8a27cd9-ca7e-4a5f-aaa7-fb2e10a21db3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.378945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6368f390-bb22-4d11-84f6-08b115709e7c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.402006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6920b017-51a5-4281-bfe9-c1c42bbe157d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.403706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:57.404096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:57.404457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:11:57.404831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dcd9d026-2e1c-4537-991d-e3e79d3864ed) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:11:57.426789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:11:57.427499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b549a20-0fe9-42a9-840a-3f0dac2defff]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.432230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be8cb75f-1eda-472d-8d8c-b92595f0271f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.433576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dcd9d026-2e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:57.486040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dcd9d026-2e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:57.486961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:57.487628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:57.488411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:57.488955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dcd9d026-2e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dcd9d026-2e1c-4537-991d-e3e79d3864ed', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4a:66:f4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:57.558713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2626de9-09ca-40ca-9968-44e7604dda27]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.596546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[75413d33-e5d8-44b9-a38a-fbac3baff665]: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:11:57.596546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:11:57.596546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:11:57.596546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:11:57.596546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:11:57.596546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:11:57.596546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:11:57.596546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:11:57.596546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:11:57.596546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:11:57.596546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:11:57.596546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:11:57.596546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:57.597559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75413d33-e5d8-44b9-a38a-fbac3baff665]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:57.597837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:57.599721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:57.602127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16d60b7b-4c0f-433c-b2dc-f56ef4d39107]: (4, ['qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.602127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:57.602345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:57.602467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:11:57.602813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:57.603913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aedada7a-2e69-4418-8acd-ffe869004bc3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.604924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1bd2875-45ba-4a8c-bc07-89448aaac5b6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.606044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57594c2b-6a0b-4677-977b-fb7924e06db4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.631989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05a32b7e-7131-4a45-ac6b-b95718a63bd2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 521356, 'reachable_time': 15756, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179473, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.639407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:11:57.639654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19009892-ec63-449a-bb9c-aae26a344d9e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:57.640859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerDiskConfigTestJSON-1194626940-router", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171" Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:40Z", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:30:21:9b", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4141, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:42Z", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dcd9d026-2e1c-4537-991d-e3e79d3864ed", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "734ab8c1-e6b3-4c59-a80b-0d64c2042a5d", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4a:66:f4", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4145, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:40Z", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:53Z", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:11:57.644292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 0c643ec4-5675-470c-b48c-12af8e5d7119. Wait time elapsed: 16.078 Jul 01 10:11:57.659118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 0c643ec4-5675-470c-b48c-12af8e5d7119. Wait time elapsed: 16.078 Jul 01 10:11:57.659118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:11:58.572110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.929s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:11:58.572769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:11:58.574509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (fd3eae0a-8937-45ea-8cb4-ab69b5ec790b), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:11:58.574843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:58.575144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:58.576364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d586abfe-7fb9-4d17-a714-149b2134002b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.592664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09596fd7-67e1-468d-a412-29944018da52]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.619044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2a4393d-184f-4ee1-adf5-b73bab408409]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.654388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96284c2f-e5f0-44e0-a898-74c0c398d740]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 179479, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.678235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29041df5-59df-4d8a-ad22-9b017f2e8333]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.706366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a78125f-6a1a-4186-9205-36a9162fda6a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.735333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[408e733f-dd99-4c3f-8393-716cf01ff9da]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.760440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65b73a2b-482e-460f-9569-410108c64cce]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.761191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:11:58.762131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:11:58.763019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:11:58.763574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dcd9d026-2e1c-4537-991d-e3e79d3864ed) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:11:58.791889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:11:58.792642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[944b414d-ff69-4d6b-af7e-6da974057a71]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.796281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1220b336-7cf5-490d-9100-2ab27ddd253a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.797622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dcd9d026-2e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:58.836680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dcd9d026-2e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:58.837597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:58.838701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:58.839246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:58.839861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dcd9d026-2e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dcd9d026-2e1c-4537-991d-e3e79d3864ed', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4a:66:f4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:11:58.900759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e04205de-b9db-4c65-ae6e-ffd524e77f7b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.965701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c5ec54e5-9998-420d-b5e1-4654b44030e1]: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:11:58.965701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:11:58.965701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:11:58.965701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:11:58.965701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:11:58.965701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:11:58.965701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:11:58.965701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:11:58.965701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:11:58.965701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:11:58.965701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:11:58.965701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:11:58.965701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:58.967703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5ec54e5-9998-420d-b5e1-4654b44030e1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:58.969330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:58.972634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:58.977483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ca501ab-78b4-4776-9a43-4df0c78f09bf]: (4, ['qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.978535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:58.979038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:58.979841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:11:58.980334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:11:58.981558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7e957e2-6dbc-4493-baa6-8a33361ce5f0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.982984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c726a521-7ddd-4f3b-83ea-d33950456606]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:58.984112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da92561c-3956-4ff8-8f69-cb42902cc9c2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:59.034235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a249a67-dfea-413c-965a-fe81cd9d71a7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 521491, 'reachable_time': 32476, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179500, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:59.041986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:11:59.043121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[873fcf6c-7d54-48fe-8b74-211fe6fd21d3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:11:59.045008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerDiskConfigTestJSON-1194626940-router", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171" Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:40Z", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:30:21:9b", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4141, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:42Z", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dcd9d026-2e1c-4537-991d-e3e79d3864ed", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "734ab8c1-e6b3-4c59-a80b-0d64c2042a5d", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4a:66:f4", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4145, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:40Z", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:57Z", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 13, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:11:59.051242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:11:59.065633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 6b447daf-1f0a-4859-816d-d4ff632e4ef2. Wait time elapsed: 19.265 Jul 01 10:11:59.066815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 6b447daf-1f0a-4859-816d-d4ff632e4ef2. Wait time elapsed: 19.268 Jul 01 10:11:59.066815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:12:01.629692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.563s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:12:01.630302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:12:01.630859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (fd3eae0a-8937-45ea-8cb4-ab69b5ec790b), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:12:01.631181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:12:01.631497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:12:01.633684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39b5d1f7-903d-484f-b514-65fd0ac43746]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.649775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52154a71-e008-4a1a-971a-cfe7e78bd3d0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.672152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c3761cb-1efa-4de1-8359-a3afe0c892d7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.705497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a7a4138-e447-4849-8d33-eafa3a413c74]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 179552, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.732375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08f889a6-891a-4c01-bcad-ba86acc11ef3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.754973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e444152-9de6-4940-8592-f3449fa02866]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.781590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2058f13e-4616-481c-a909-611caac642ff]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.805441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5062ea98-e539-4b64-824c-48277e475297]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.806190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:12:01.806479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:12:01.806738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:12:01.807898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dcd9d026-2e1c-4537-991d-e3e79d3864ed) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:12:01.825364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:12:01.825831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b217d4c1-5cbc-4c32-9bd2-4ef76130c02e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.829243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[969a250d-a27c-463f-bff4-aa5af5471f09]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.830500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dcd9d026-2e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:01.871396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dcd9d026-2e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:01.872006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:01.872411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:01.872806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:01.873195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dcd9d026-2e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dcd9d026-2e1c-4537-991d-e3e79d3864ed', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4a:66:f4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:01.918784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[561265dc-ee23-4bff-bae2-016533918cd1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.949851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f4841d56-dd5c-4e3f-9c68-f56e52b5ddfe]: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:12:01.949851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:12:01.949851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:12:01.949851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:12:01.949851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:12:01.949851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:12:01.949851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:12:01.949851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:12:01.949851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:12:01.949851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:12:01.949851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:12:01.949851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:12:01.949851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:01.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4841d56-dd5c-4e3f-9c68-f56e52b5ddfe]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:01.951651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:01.953753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:12:01.957663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[daa458b1-46f4-4095-8912-4470595537e0]: (4, ['qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.957663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:12:01.957663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:12:01.957663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:12:01.957663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:12:01.958597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08f48a52-3554-48b3-b1ea-aefe11f6dae0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.959652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[099dca14-837f-4d69-9ae1-3f46d7e833da]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.961178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00cc5631-3a43-475d-913e-824370578a60]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.980741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c141f87-5bbb-4ec2-a190-9dc1b2c67ad5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 521796, 'reachable_time': 26468, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179570, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.986342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:12:01.986342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a6d00c0-ec09-459c-86ee-e266485883e3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:01.987350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerDiskConfigTestJSON-1194626940-router", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171" Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:40Z", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:30:21:9b", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4141, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:42Z", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dcd9d026-2e1c-4537-991d-e3e79d3864ed", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "734ab8c1-e6b3-4c59-a80b-0d64c2042a5d", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4a:66:f4", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4145, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:40Z", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:12:00Z", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:12:01.990505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 0c643ec4-5675-470c-b48c-12af8e5d7119. Wait time elapsed: 20.425 Jul 01 10:12:02.003081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 0c643ec4-5675-470c-b48c-12af8e5d7119. Wait time elapsed: 20.425 Jul 01 10:12:02.003081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:12:02.774648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.785s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:12:02.775909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:12:02.776944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (fd3eae0a-8937-45ea-8cb4-ab69b5ec790b), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:12:02.777851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:12:02.778195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:12:02.779446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[656eccff-42f5-45e7-96b5-1fcef1e47d1c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:02.796592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e5976b0-f3d7-400b-996e-40e4283332e9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:02.818131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b65ae68-1853-45e5-aeab-21f15f797266]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:02.851727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[641b366b-be0f-4b12-88b0-30e0de4164f8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 179573, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:02.875528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf32c6c3-1764-4210-86c1-479cb6973816]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:02.898824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7610e1d0-0935-4f0b-a0e1-da2764eeb3d8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:02.924424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a247223-7fdc-4e7b-ac92-3b242c539e79]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:02.946748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0015ad84-2f5a-4a18-bc85-c5b56cd3e30c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:02.947777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:12:02.948077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:12:02.949182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:12:02.949472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dcd9d026-2e1c-4537-991d-e3e79d3864ed) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:12:02.967983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:12:02.969543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91965961-1613-4fa6-a5a6-705ab3c065e5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:02.973232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97828928-e49b-406e-81e7-c5c51800dd83]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:02.974171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dcd9d026-2e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:03.015466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dcd9d026-2e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:03.016378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:03.016774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:03.017178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:03.017527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dcd9d026-2e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dcd9d026-2e1c-4537-991d-e3e79d3864ed', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4a:66:f4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:03.055187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a47d8da0-0623-470a-aa37-8a3eda21ebc7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:03.114545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b406164b-cb73-4c2e-9f64-50e25791e912]: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:12:03.114545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:12:03.114545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:12:03.114545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:12:03.114545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:12:03.114545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:12:03.114545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:12:03.114545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:12:03.114545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:12:03.114545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:12:03.114545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:12:03.114545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:12:03.114545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:03.115573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b406164b-cb73-4c2e-9f64-50e25791e912]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:03.115784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:03.117629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:12:03.123149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[daf5960f-a8b0-42b7-8183-b254456ecb33]: (4, ['qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:03.123149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:12:03.123149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:12:03.123149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:12:03.123149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:12:03.123149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f06431e-b434-4df1-a241-5aab974e11a6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:03.124625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73a863e3-f395-47c7-8fa4-9648512e2c93]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:03.126002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55c39caa-c1cd-44e6-ae45-d13da00226a8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:03.146197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a52899c-d2a9-4c79-a284-0ab9eef6e97c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 521911, 'reachable_time': 15108, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179593, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:03.152105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:12:03.152557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb675f88-e67a-4674-af7c-9fc66542122d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:03.153628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerDiskConfigTestJSON-1194626940-router", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171" Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:40Z", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:30:21:9b", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4141, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:42Z", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dcd9d026-2e1c-4537-991d-e3e79d3864ed", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "734ab8c1-e6b3-4c59-a80b-0d64c2042a5d", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4a:66:f4", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4145, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:40Z", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:12:02Z", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:12:03.156429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 6b447daf-1f0a-4859-816d-d4ff632e4ef2. Wait time elapsed: 23.358 Jul 01 10:12:03.169644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 6b447daf-1f0a-4859-816d-d4ff632e4ef2. Wait time elapsed: 23.358 Jul 01 10:12:03.169644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:12:05.745662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.589s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:12:05.746257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:12:05.746903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (fd3eae0a-8937-45ea-8cb4-ab69b5ec790b), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:12:05.747225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:12:05.747529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:12:05.748707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f827a9c-a715-4e5e-a282-e2b2403ad537]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:05.759740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6dfa7b84-63cc-4e22-b4ea-df258e5f09da]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:05.781500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49ab0a43-1600-4ba4-99eb-8c7dc0cd7c6a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:05.816720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1ab42e1-274c-4a57-a976-91b0c1121823]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 179598, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:05.837964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8282ee0-05ae-41d8-b613-0bc77b20401a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:05.858407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[084aa6c3-514b-4e1d-b6f5-128fe4cb8fa6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:05.880268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3cfaec9-ecf7-469e-9f7e-f4ec7c6d6ae0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:05.905186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41ac36f7-b42a-4e0a-ab33-ec5ee97013fd]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:05.906387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:12:05.906994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:12:05.907553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:12:05.907984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dcd9d026-2e1c-4537-991d-e3e79d3864ed) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:12:05.926462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:12:05.926755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1a9f297-efe0-461d-b977-c7e4e4258699]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:05.930790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22629ef7-3977-4344-a08a-26930f1e9dd9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:05.931776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dcd9d026-2e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:05.985755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dcd9d026-2e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:05.986796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:05.987387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:05.987957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:05.988540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dcd9d026-2e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dcd9d026-2e1c-4537-991d-e3e79d3864ed', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4a:66:f4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:06.056210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f2b77a4-b58f-4430-b107-426de5fb165a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:06.108637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f8b29f51-3e52-4602-9311-2e3ee6112f1b]: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:12:06.108637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:12:06.108637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:12:06.108637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:12:06.108637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:12:06.108637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:12:06.108637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:12:06.108637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:12:06.108637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:12:06.108637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:12:06.108637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:12:06.108637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:12:06.108637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:06.110468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8b29f51-3e52-4602-9311-2e3ee6112f1b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:06.111640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:06.114537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:12:06.119502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35092483-6342-4ddc-ab8a-264bf1a0b5b0]: (4, ['qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:06.121381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:12:06.122010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:12:06.122573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:12:06.123062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:12:06.128930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe01c5ea-3cfe-449e-89ca-d43a46028c04]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:06.136089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcc1e7fd-6c79-4458-ad87-4e79d1807bb5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:06.137489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe0e6dc6-447a-4b26-a2d7-4d605329c65e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:06.199238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc8c4cc3-a39d-41d5-9b34-370f5acb4447]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 522208, 'reachable_time': 16468, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179640, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:06.238184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:12:06.243128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de0dd78e-cb0f-4e97-bf2e-8f4dc67365ac]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:06.246074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerDiskConfigTestJSON-1194626940-router", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171" Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:40Z", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:30:21:9b", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4141, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:42Z", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dcd9d026-2e1c-4537-991d-e3e79d3864ed", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "734ab8c1-e6b3-4c59-a80b-0d64c2042a5d", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4a:66:f4", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4145, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:40Z", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:12:03Z", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:12:06.251278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:12:06.264178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3 Jul 01 10:12:06.265258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 0c643ec4-5675-470c-b48c-12af8e5d7119. Wait time elapsed: 24.700 Jul 01 10:12:06.265258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 2, update_id 0c643ec4-5675-470c-b48c-12af8e5d7119. Wait time elapsed: 24.701 Jul 01 10:12:06.265258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:12:06.995076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.730s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:12:06.995691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:12:06.996421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (fd3eae0a-8937-45ea-8cb4-ab69b5ec790b), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:12:06.996827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:12:06.997119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:12:06.998867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eee0f1ba-9cb2-478d-a28d-3d52580ea87f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.013296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cec67c5-6e7a-4f4d-b9f5-321efbe5ffc8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.046509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61783396-cd82-46c5-b1be-d803101c1fca]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.074511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[054a813d-f432-4b63-b29f-915d86a7da74]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 179668, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.094925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ac687c1-6107-4eea-8ab0-2f17686ef942]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.115330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67b69f71-879d-41e1-bc18-80206b9d61aa]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.135756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c891546-a92a-4910-b0a0-14396a8dac86]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.156437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a65c5e18-cffc-4ecb-ba8b-0371a21d640c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.157119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:12:07.157457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:12:07.157733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:12:07.158106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dcd9d026-2e1c-4537-991d-e3e79d3864ed) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:12:07.173627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:12:07.174013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0051b76-187b-40a2-a8b5-8925d032ca67]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.176932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c39af69f-c092-4733-8dbf-aff0e8a6f9f7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.178179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dcd9d026-2e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:07.218854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dcd9d026-2e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:07.219560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:07.219959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:07.221010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dcd9d026-2e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:07.221407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dcd9d026-2e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dcd9d026-2e1c-4537-991d-e3e79d3864ed', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4a:66:f4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:12:07.272665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e02e648b-7979-457a-8896-71d201f96aff]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.302252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5fb5a672-5771-45d1-af58-4a84dff6aaba]: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:12:07.302252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:12:07.302252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:12:07.302252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:12:07.302252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:12:07.302252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:12:07.302252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:12:07.302252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:12:07.302252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:12:07.302252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:12:07.302252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:12:07.302252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:12:07.302252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:07.303354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fb5a672-5771-45d1-af58-4a84dff6aaba]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:07.303559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:07.305346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:12:07.307876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48fbdcad-e309-496f-aba7-3a1096198597]: (4, ['qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.307876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:12:07.307876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:12:07.307876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:12:07.307876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/fd3eae0a-8937-45ea-8cb4-ab69b5ec790b.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:12:07.308924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79384366-a51a-417b-972c-14471fccc643]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.309983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3276fdd2-6f75-45e8-b7db-f8fd97d2a4b3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.310956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50db9fc9-3b00-47c1-8c40-14b85e36e7e6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e68adb9f-dc5f-4552-b0c7-11cceb021a79]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 522333, 'reachable_time': 42876, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 179686, 'error': None, 'target': 'qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.340245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:12:07.340502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4157d7df-4bce-4365-bda8-c042795f43d3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: fd3eae0a-8937-45ea-8cb4-ab69b5ec790b: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dcd9d026-2e not found in namespace qrouter-fd3eae0a-8937-45ea-8cb4-ab69b5ec790b. Jul 01 10:12:07.341525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerDiskConfigTestJSON-1194626940-router", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171" Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:40Z", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "731d77cc-7fd5-4dfb-b649-2ab8c0d0f0a1", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:30:21:9b", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4141, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.171", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:38Z", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:11:42Z", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dcd9d026-2e1c-4537-991d-e3e79d3864ed", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "734ab8c1-e6b3-4c59-a80b-0d64c2042a5d", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4a:66:f4", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "fd3eae0a-8937-45ea-8cb4-ab69b5ec790b", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4145, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:11:40Z", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:12:05Z", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 21, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c1207b239e0e423cb4c2ffdf1aa9d441", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dea772ae-92e9-4224-88fe-14663c5713b8", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:12:07.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3 Jul 01 10:12:15.119940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:12:15.120607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:12:15.120607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:12:15.120607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:12:15.232301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:12:55.233940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:13:15.120843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:15.121545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:15.121545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:15.121742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:29.304441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-213213c9-ab6e-4657-bbc7-d2f68438fff1 admin admin] Got routers updated notification :['fd3eae0a-8937-45ea-8cb4-ab69b5ec790b'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:13:29.305069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 1, update_id b821b79d-f378-446e-a091-f723d71d7fe6. Wait time elapsed: 0.000 Jul 01 10:13:29.305418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 3, priority 1, update_id b821b79d-f378-446e-a091-f723d71d7fe6. Wait time elapsed: 0.001 Jul 01 10:13:29.305783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:13:30.092677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-39c2f46e-ee39-4aa5-ae35-922543835b7c admin admin] Got router deleted notification for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:13:30.512900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.207s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:13:30.513553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b was not found. Performing router cleanup Jul 01 10:13:30.516032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[489551bd-b3be-4cba-a2f7-76af08d6662a]: (4, ['qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:30.517023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:30.517312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:30.517569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, update_id b821b79d-f378-446e-a091-f723d71d7fe6. Time elapsed: 1.213 Jul 01 10:13:30.518012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 1, priority 1, update_id f38710e7-0c59-4aa8-9538-9623e1d7f94c. Wait time elapsed: 0.425 Jul 01 10:13:30.518269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, action 1, priority 1, update_id f38710e7-0c59-4aa8-9538-9623e1d7f94c. Wait time elapsed: 0.425 Jul 01 10:13:30.518502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router fd3eae0a-8937-45ea-8cb4-ab69b5ec790b was not found. Performing router cleanup Jul 01 10:13:30.520261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbd6f1dd-9930-488f-940e-dd8f184cacb5]: (4, ['qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:30.520914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:30.521180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:30.521974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for fd3eae0a-8937-45ea-8cb4-ab69b5ec790b, update_id f38710e7-0c59-4aa8-9538-9623e1d7f94c. Time elapsed: 0.004 Jul 01 10:13:34.804149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-48ec1f83-9580-4bbc-8382-e543d4bc54ff admin admin] Got routers updated notification :['f0193d7f-79a0-4dc0-b92f-43b1cb63c572'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:13:34.804795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 1, update_id 4fef0a19-d1df-4942-a576-b40c6e1079bf. Wait time elapsed: 0.000 Jul 01 10:13:34.805272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 1, update_id 4fef0a19-d1df-4942-a576-b40c6e1079bf. Wait time elapsed: 0.001 Jul 01 10:13:34.805839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:13:35.236923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:13:36.222485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.416s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:13:36.223093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:13:36.225705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:13:36.226327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:36.226710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:36.229082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ef4d768-f467-4bfd-ac4e-75b3c3735af0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.284318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f0fc191-88be-481f-b47c-41f6c7a8e59b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.320816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f12cddd-4fb1-4944-8c5b-b90e86604e48]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.373811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40e85905-4d93-48fe-ae74-3df57415643f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181049, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.440992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43272d7c-0ef6-484f-a71a-cf91d63961db]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.479733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c9bf271-d1a3-4f5e-bc5f-56d744ea8a85]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.527341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7874fbae-c2a6-40f4-8d77-ea5e7c46153e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.553257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[281b3ef4-c33f-4d76-b033-94c8a02a300c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.553990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:36.554474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:36.554758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:13:36.575754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1acfa332-dd97-40a6-bad9-99cd6d0fe92c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 531263, 'reachable_time': 25548, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181069, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.577340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:36.577664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:36.578107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:13:36.578421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:13:36.578764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:36.579086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:36.579641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '6f31e69d-f38a-4748-82eb-082fe6e029b9', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:0b:81:f4', 'admin_state_up': True, 'status': 'DOWN', 'device_id': 'f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'device_owner': 'network:router_gateway', 'standard_attr_id': 4259, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.220', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:13:33Z', 'updated_at': '2022-07-01T10:13:35Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-6f31e69d-f3), ns(qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:13:36.596851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-6f31e69d-f3 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:13:36.597608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca3e90d3-d449-469b-91a2-e2751c0d4cea]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d5f150f-5e24-4f3c-9b6f-509b6ff2658e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.602125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-6f31e69d-f3, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:36.602633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:13:36.603642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-6f31e69d-f3, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:36.604505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-6f31e69d-f3, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:36.605098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-6f31e69d-f3, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:36.605662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-6f31e69d-f3, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:36.606222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-6f31e69d-f3, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6f31e69d-f38a-4748-82eb-082fe6e029b9', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:0b:81:f4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:36.656663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f6f6d94-4267-4bd3-ab42-7e6e3b86321a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.699869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b5c147f7-ff16-4f9e-b281-6acf394439ad]: Network interface qg-6f31e69d-f3 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:13:36.699869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:13:36.699869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:13:36.699869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:13:36.699869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:13:36.699869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:13:36.699869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:13:36.699869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:13:36.699869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:13:36.699869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:13:36.699869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:13:36.699869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:13:36.699869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f31e69d-f3 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:36.700725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5c147f7-ff16-4f9e-b281-6acf394439ad]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-6f31e69d-f3 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.700725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:13:36.701574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:13:36.709741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-914c7ee1-d6f2-426a-840e-122f7b6cc028 admin admin] Got routers updated notification :['f0193d7f-79a0-4dc0-b92f-43b1cb63c572'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:13:36.721948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a6b0e1a-6132-48d0-80c9-8c1601ae96c1]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.902791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[395a0410-0aaa-4440-98de-e6b30a47235c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:36.915627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80d4e766-975e-4956-b579-60f7f5c5b773]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:37.070207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f1c76ab-f7b1-4bb9-8db7-f9d359e70b4c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:37.070782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:13:37.071094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:13:37.071418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:13:37.071863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572" released by "process_external" :: held 0.493s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-6f31e69d-f3 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f31e69d-f3 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f31e69d-f3 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:37.072746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f31e69d-f3 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f31e69d-f3 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:37.074924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:37.077669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b444d0e4-ac0a-451d-afab-92e03a1a6364]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:37.077669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:37.077669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:37.077669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:13:37.077669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:37.077669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2ce567a-932e-4e69-9983-35d6408203e8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:37.080004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9a7e69f-df8e-4470-9a25-c1a30ca3263d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:37.080004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9c76ba6-24d7-412c-a78b-a736385e0b07]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:37.107647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4bb2d77-120b-4465-94ac-d1b27a57e4f2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 531263, 'reachable_time': 25548, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181085, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:37.115464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:13:37.115982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[303433ff-3e14-49a7-a148-e1e4eb02fad4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f31e69d-f3 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6f31e69d-f3 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:37.119198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:33Z", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 1, update_id efa4df34-2a81-438d-b96e-030f25c81e10. Wait time elapsed: 0.410 Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 1, update_id efa4df34-2a81-438d-b96e-030f25c81e10. Wait time elapsed: 0.410 Jul 01 10:13:37.122893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:13:41.644364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.524s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:13:41.644978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:13:41.645663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:13:41.646230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:41.646523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:41.647830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b5faf4a-2453-4030-81f8-bb3f51a5294f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.664040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0975f43f-b598-43ac-a516-7c44c318df67]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.691952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[874e3480-8fbc-45f8-9cce-b874d7b294b0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.728364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16f7538d-70bb-4c30-ba28-6a6fe7af3a73]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181139, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.751522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e86b4d39-3566-475b-a76c-843232d2ba42]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.777934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a038d89-a20f-469c-8e0b-5efc007eb9ce]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.803912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54f3352e-e537-46c9-908a-30105ee515c6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.825820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3aa6d4c1-1d82-4b71-9baf-e2c6e9afef2e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.826849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:41.827250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:41.827591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:13:41.827876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:13:41.846535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:13:41.846855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9bb77fe-1e61-4312-a28e-2e5a716b9752]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.851016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f6e221d-fd52-4b61-b6b3-a120e358d6bd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.852151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:41.852544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:13:41.853915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:41.854691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:41.855145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:41.855858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:41.856448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:41.902028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75f2b008-ce8c-4215-989c-41cba1f05d5d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.937629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6a600870-ddc8-405a-8415-1d4dfc55f4ee]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:13:41.937629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:13:41.937629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:13:41.937629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:13:41.937629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:13:41.937629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:13:41.937629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:13:41.937629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:13:41.937629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:13:41.937629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:13:41.937629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:13:41.937629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:13:41.937629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:41.938568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a600870-ddc8-405a-8415-1d4dfc55f4ee]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:41.939208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:41.941201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:41.943664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dda93d85-f35b-4e21-9c4c-d3b262ca2f34]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.943664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:41.943664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:41.943664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:13:41.943664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:41.945297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8b4b9f8-b040-403e-988c-d96cfceacadc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.946319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da0dc2fd-b2e5-4780-aea7-a09a8e00c85d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.948050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d45ee36b-ec5d-48c8-9370-caa45810cd85]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.973806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48c85502-1ac4-46e7-aa49-45fdb38c2347]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 531798, 'reachable_time': 24392, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181156, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.976015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:13:41.976488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01265a36-7652-402a-8bdd-d85064b81385]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:41.980058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:13:41.984904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:13:42.001097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4fef0a19-d1df-4942-a576-b40c6e1079bf. Wait time elapsed: 7.181 Jul 01 10:13:42.001097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4fef0a19-d1df-4942-a576-b40c6e1079bf. Wait time elapsed: 7.181 Jul 01 10:13:42.001097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:13:42.844999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.859s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:13:42.846178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:13:42.847998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:13:42.848710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:42.849201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:42.851321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c25f9766-b6ca-4883-b7b3-ff5dcaef3adb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:42.867440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6b7714b-87b6-4f10-97e2-a4148f5b2f3d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:42.903409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[693a4628-831a-4fa3-8418-0d2d6a3466a6]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:42.931952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0613772e-b4b6-4330-924d-1e964e39e5dc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181160, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:42.952469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8937470-32a0-4335-85c7-0a4bb7be58d7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:42.972519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1508d71e-ebeb-4e45-915b-bc185c37bdc7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:42.997607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[558910b0-a6ac-48ff-9e42-81f138f8385e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:43.017486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2feb156-89bb-4586-9916-1ef3407231c6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:43.018045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:43.018316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:43.018579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:13:43.018948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:13:43.034768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:13:43.035023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa8f6d8c-ccd2-4040-aa34-6c33c9369db6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:43.037865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12e01db0-bec6-488f-9a2c-c37415085663]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:43.038802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:43.082307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:43.083060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:43.083930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:43.084463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:43.084849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:43.129614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ea210d4-3dc6-4175-8033-c3dc28b70ce5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:43.158150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3934f227-dc03-4b4f-b4c9-9b3824bc8e2f]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:13:43.158150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:13:43.158150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:13:43.158150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:13:43.158150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:13:43.158150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:13:43.158150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:13:43.158150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:13:43.158150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:13:43.158150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:13:43.158150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:13:43.158150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:13:43.158150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:43.159210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3934f227-dc03-4b4f-b4c9-9b3824bc8e2f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:43.159301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:43.160928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:43.162966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[817cb94a-0516-4311-8bac-ed1b87f8f853]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:43.162966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:43.162966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:43.162966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:13:43.163289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:43.164310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83fed609-fcb9-4c57-9668-fe107c59e7c4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:43.165249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09a9c162-4b4f-4a74-9f51-1d291fd8b129]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:43.166202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21e74672-c5fc-4d22-946c-807859590fdc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:43.184983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[583f9cee-2671-4303-9eca-50013326e298]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 531919, 'reachable_time': 44124, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181178, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:43.187152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:13:43.187609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ab1e9c4-639e-49ae-ad91-b49da31d5ff9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:43.188738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:13:43.193289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:13:43.206812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id efa4df34-2a81-438d-b96e-030f25c81e10. Wait time elapsed: 6.483 Jul 01 10:13:43.206812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id efa4df34-2a81-438d-b96e-030f25c81e10. Wait time elapsed: 6.483 Jul 01 10:13:43.206812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:13:45.909042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.715s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:13:45.909681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:13:45.910402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:13:45.910739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:45.911024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:45.912281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2a4e1d6-dde6-40a1-a8d8-1de5281f9449]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:45.924375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9ad253e-2c9c-4337-859a-683db0c7e240]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:45.946141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2096992d-21f2-48e7-91df-646683ac8522]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:45.975643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a824fef1-be8c-4dc2-bb2b-544bcdeeee48]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181228, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:45.997309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2dba34c2-0bcb-4389-837c-fc9b07467c82]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.017085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38c0139f-b5c3-415f-9113-8b7c272776fd]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.036593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5670d131-04cf-4e88-bb7c-af1183c5528d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.056976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67bfdbe2-f8be-452f-8760-ca585080b79f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.057519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:46.057813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:46.058075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:13:46.058353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:13:46.073170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:13:46.073428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1e5d8ab-7109-4aa4-91f2-460d4b05fb63]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.076309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abbd34f7-6a42-46e7-a86f-41df099875f4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.077281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:46.106821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:46.107954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:46.108435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:46.109125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:46.109579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:46.162938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e23494e1-074e-42d2-a06a-1d8e82f82f2a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2f79c283-feb6-4a21-9597-75dda48809e0]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:13:46.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:13:46.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:13:46.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:13:46.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:13:46.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:13:46.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:13:46.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:13:46.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:13:46.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:13:46.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:13:46.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:13:46.191276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:46.192383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f79c283-feb6-4a21-9597-75dda48809e0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:46.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:46.194266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:46.196461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73b09c49-51dd-4dfe-9981-33375c7190f7]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.196461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:46.196461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:46.196461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:13:46.196461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:46.197362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1780237d-1c3a-4cda-b210-c212be97eb6f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.198483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac5e5d74-1b85-4389-aaab-076b95c39ff9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.199470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac051ebc-fceb-4e9b-b58d-2a128ed803f8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.218823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c249f19-fa04-4975-8ae2-8bc78c18fc2a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 532224, 'reachable_time': 40148, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181246, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db1c573d-5dc3-4777-b82c-9517a9d5da77]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:46.222988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:42Z", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:13:46.226435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4fef0a19-d1df-4942-a576-b40c6e1079bf. Wait time elapsed: 11.420 Jul 01 10:13:46.245256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4fef0a19-d1df-4942-a576-b40c6e1079bf. Wait time elapsed: 11.421 Jul 01 10:13:46.245256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:13:46.942957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.718s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:13:46.943871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:13:46.948345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:13:46.948682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:46.948996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:46.950258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e42f105-c467-42b7-921b-805349906928]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.966955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fad19cc1-612e-4780-8f32-f8474d9f810a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:46.988909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ea1ad47-842b-466a-bad5-7739da32a40e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.017731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3d26dc2-0f1b-4912-af69-f4459327ec4e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181251, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.038462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80e8988c-fe5f-4d71-8722-76e41c61323f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.064985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0125ca05-fe10-460a-8398-fd144d9b750a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.087129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dff1ea65-aeb1-40bc-8446-eee549ef7b92]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.108442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6bde39ce-3271-42c6-8316-b5af264a7692]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.109116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:47.109394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:47.109661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:13:47.109940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:13:47.126321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:13:47.126608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a117ee25-4842-4e41-b4d4-e43837f728b3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.129216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ef6673f-26b8-461a-84f6-1b6bcfbc5ac2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.130170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:47.187737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:47.188418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:47.188815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:47.189200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:47.189575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:47.236086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b8abb29-b691-4fae-83a8-09ac6969d3ca]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1d4f12f9-971c-45dd-a787-0ca9170beb88]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:13:47.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:13:47.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:13:47.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:13:47.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:13:47.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:13:47.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:13:47.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:13:47.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:13:47.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:13:47.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:13:47.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:13:47.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d4f12f9-971c-45dd-a787-0ca9170beb88]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:47.274966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:47.277399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:47.279554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c26eba5-79d7-440b-90f2-b496145c7cce]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.279554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:47.279554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:47.279554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:13:47.279554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:47.280059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b493ddde-2608-4669-8602-6954b228340e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.280895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5277a4ca-b2dc-49d9-9c61-1641cc6bf27a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.282203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[073567c7-f136-43f4-a300-343974b7685c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.304508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f7b0b1d-e4b2-4b6d-b6e8-18693995556f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 532328, 'reachable_time': 24288, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181270, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.307515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:13:47.307930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f64c4ed-ccf2-4ec6-8ad7-2ef55c6e1058]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:47.309098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:45Z", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:13:47.313529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:13:47.328962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id efa4df34-2a81-438d-b96e-030f25c81e10. Wait time elapsed: 10.614 Jul 01 10:13:47.328962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id efa4df34-2a81-438d-b96e-030f25c81e10. Wait time elapsed: 10.614 Jul 01 10:13:47.328962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:13:49.492326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.168s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:13:49.492979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:13:49.493535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:13:49.494075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:49.494353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:49.495752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[600da155-8b8e-4436-94ef-f3fadba3bce7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.510472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd2659e6-94e4-4dd3-9815-ff82c4f26f89]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.531569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37575f15-193d-441c-bf98-b0627b698ce8]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.560758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04f1e026-3e8d-42de-a070-4ddfa5811418]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181275, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.582227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf454fcf-f10a-4c3d-bcf4-464fb2f98020]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.604679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fff9f38-3232-4016-985c-565b7cd707e6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.627155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc0e4a39-298f-433c-84a8-50d0a5d75e11]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.647500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a162e76-5b02-47b9-ba8d-6bc59667de75]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.648200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:49.648507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:49.648798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:13:49.649564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:13:49.666270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:13:49.666749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9453b3f-e884-43ad-9b3d-ca113b71c015]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.669534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[510d0683-9bd2-4663-8bca-5ab15b67a7cd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.670518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:49.706080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:49.707096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:49.707345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:49.707850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:49.708267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:49.757487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7a5ab73-420c-41bf-aa74-867fd3d810a6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.795024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[dbf4dfb3-11ce-4e3e-b930-e6c281f18fe5]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:13:49.795024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:13:49.795024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:13:49.795024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:13:49.795024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:13:49.795024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:13:49.795024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:13:49.795024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:13:49.795024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:13:49.795024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:13:49.795024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:13:49.795024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:13:49.795024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:49.796295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbf4dfb3-11ce-4e3e-b930-e6c281f18fe5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:49.796720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:49.798671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:49.800926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f79002a6-5604-4b7b-ac2a-07b1bc9b2bad]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.800926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:49.800926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:49.801780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:13:49.802290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:49.803582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17fc0aaf-dcda-4090-8799-9814fc0ad5cd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.805106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[166fc05f-cb06-4d66-b9b3-494097dbbdc8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.807186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ea40cae-0388-44b0-ae22-70f34f00f722]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.828646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3cfa1e33-6cb5-4f17-9607-b67319555b4e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 532582, 'reachable_time': 26564, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181294, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.836066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:13:49.836355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f7d2c27-c0c4-4219-be13-65b981999d17]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:49.837466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:47Z", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:13:49.842979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:13:49.864366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4fef0a19-d1df-4942-a576-b40c6e1079bf. Wait time elapsed: 15.055 Jul 01 10:13:49.864366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4fef0a19-d1df-4942-a576-b40c6e1079bf. Wait time elapsed: 15.055 Jul 01 10:13:49.864366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:13:50.668792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.809s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:13:50.669458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:13:50.670245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:13:50.670627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:50.670966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:50.672247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37d9e09b-31cc-4007-8d04-3b891bc81fe0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:50.684803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f24c16c1-479d-48f5-965d-fb512d3d4e3e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:50.706694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33f6fbd5-d65a-4336-b7cd-1bda32f3c4f1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:50.734598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[981c9761-ced1-4443-a117-93059325f5a4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181298, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:50.756528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0068ca53-ce91-4def-b9ab-0dedf0f1c901]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:50.778524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e04605df-4421-4ce2-b766-fed39a157f32]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:50.800596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18e96e73-6ace-485d-b519-fea59da829f8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:50.823189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6743bac-5719-4c74-83b5-6db25480e1d2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:50.824206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:50.824633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:50.825060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:13:50.826339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:13:50.846213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:13:50.846577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00025a1e-bab1-4724-b6e3-0e42abcc87c4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:50.849797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[452dbcc8-45b9-439a-9870-0baacd4de184]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:50.850664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:50.889210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:50.891612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:50.892294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:50.894462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:50.895036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:50.958426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc753ab3-c330-4da8-89c7-1d7c939a4c84]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:51.047795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f388ccc9-1e07-4b45-85d1-06ad6b7867b5]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:13:51.047795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:13:51.047795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:13:51.047795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:13:51.047795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:13:51.047795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:13:51.047795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:13:51.047795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:13:51.047795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:13:51.047795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:13:51.047795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:13:51.047795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:13:51.047795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:51.048797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f388ccc9-1e07-4b45-85d1-06ad6b7867b5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:51.049100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:51.050972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:51.055508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbc561d8-278a-4687-86a0-a30a1016473b]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:51.056677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:51.056967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:51.057238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:13:51.057588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:51.058710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d98fdaf7-f0dd-4e4e-9476-a4106fe797fa]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:51.067299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c221011-f1be-4b49-b281-b78b23748f00]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:51.068550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c98628f-3383-463b-abf8-dffb2eb5cd6c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:51.126344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18fe44b7-dd6a-434d-be5a-ca0c1dc94d8f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 532699, 'reachable_time': 23548, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181339, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:51.132683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:13:51.132843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8316dfb3-5901-4c18-aa55-fbf1dcda376e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:51.133906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:49Z", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:13:51.137902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id efa4df34-2a81-438d-b96e-030f25c81e10. Wait time elapsed: 14.425 Jul 01 10:13:51.153322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id efa4df34-2a81-438d-b96e-030f25c81e10. Wait time elapsed: 14.425 Jul 01 10:13:51.153322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:13:53.512358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.376s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:13:53.513370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:13:53.513947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:13:53.514334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:53.514773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:53.516025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52e089fd-2b86-42c3-a9c2-14c71d40da51]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.531612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbad9cb3-ea4f-41b7-b125-d0d26377e01e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.554722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c7be23e-7598-4ba0-a52e-e0d9b62fa2cd]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.593092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44b1e3a8-1534-4b5f-8a0d-532c792fd10f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181366, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.615997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[937de56e-3e68-4dff-93c9-1929de24628c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.640495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1872ada-7476-41fb-95b2-e76f24ad43b8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.669965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c81cb04-35bd-417c-814a-86777873adf2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.692029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95a99063-acda-4811-a9db-107053cc19a6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.693954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:53.694416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:53.694911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:13:53.695336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:13:53.719413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:13:53.719707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a82b6fd0-874a-4fb3-a350-1e93c3de142e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.722424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ce12975-774a-46f7-98bd-75c41ee061b2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.723542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:53.778921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:53.779714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:53.780201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:53.780642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:53.781366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:53.854171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d161df91-4dce-4ace-a4ed-844913a23c42]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.892772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[56454624-4d76-432f-825c-3acbe4cee3d3]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:13:53.892772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:13:53.892772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:13:53.892772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:13:53.892772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:13:53.892772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:13:53.892772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:13:53.892772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:13:53.892772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:13:53.892772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:13:53.892772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:13:53.892772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:13:53.892772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:53.893897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56454624-4d76-432f-825c-3acbe4cee3d3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:53.894587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:53.896533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:53.900142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52b3a7bc-9b80-4e51-896c-c53e031bc3b9]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.902110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:53.902436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:53.902693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:13:53.903029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:53.904097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34e92f7b-46e4-4ecf-a997-fe1a12da13c9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.905211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c891e60a-624d-4d51-8c75-0ea1f74aef8c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.906127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e07ad32-1312-4130-9eab-88c1e0e969df]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.930026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9db83bac-afb9-46bd-9584-383e3454c7fb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 532985, 'reachable_time': 17660, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181385, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.936021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:13:53.936021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c627411-2f69-4860-99c2-228faae5bd1b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:53.936490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:52Z", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:13:53.939232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4fef0a19-d1df-4942-a576-b40c6e1079bf. Wait time elapsed: 19.133 Jul 01 10:13:53.953921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4fef0a19-d1df-4942-a576-b40c6e1079bf. Wait time elapsed: 19.133 Jul 01 10:13:53.953921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:13:54.021771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-021efc9b-994d-4c6f-824b-1aaa59369920 tempest-ServerActionsTestJSON-621858079 tempest-ServerActionsTestJSON-621858079-project] Got routers updated notification :['f0193d7f-79a0-4dc0-b92f-43b1cb63c572'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:13:54.765624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.827s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:13:54.766238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:13:54.766955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:13:54.767472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:54.767885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:54.769518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57b6d4e9-455d-411f-b47e-2b769d3cad3d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:54.784574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bf9b117-2d58-42a5-b886-a162f7e7160c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:54.805200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c68c38b9-e3ef-436d-8291-bdef19a5a032]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:54.834586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25197c15-3b2c-41c1-840a-c7dfb2a5a19e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181388, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:54.856173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c2bde25-8f72-4555-bb40-162decfef6aa]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:54.876553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f17ded0-b60a-4340-abe5-a0ed44ffde97]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:54.898620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[687c7341-7870-495c-a5de-2ad8f51bd245]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:54.920609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec98af3f-220d-4ab7-96f2-eecef2aa7796]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:54.921227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:54.921506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:54.921774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:13:54.922171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:13:54.940368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:13:54.940550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[124e98ce-87e1-40ee-a622-0bb58b6cbdad]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:54.944513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e174448-0f3f-4c4c-9813-831e6d5b9fdb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:54.945732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:54.977256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:54.979196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:54.979929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:54.982468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:54.982870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:55.022817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39c55114-1700-4e92-8781-7307f8a4466e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:55.051819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[965d967d-1de9-4e5c-b706-edcf9a19b288]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:13:55.051819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:13:55.051819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:13:55.051819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:13:55.051819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:13:55.051819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:13:55.051819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:13:55.051819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:13:55.051819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:13:55.051819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:13:55.051819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:13:55.051819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:13:55.051819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:55.052918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[965d967d-1de9-4e5c-b706-edcf9a19b288]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:55.053328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:55.055199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:55.057867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2a25af8-c743-4ec6-adcb-3c35bb675855]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:55.057867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:55.058294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:55.058547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:13:55.058902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:55.060007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e922c9d7-03d1-4bc2-894c-e3e48a0f691d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:55.060903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ffeb640-33e7-4e4e-8a99-df26217cd08c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:55.061895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[451486c7-7f4a-46e1-b928-d5c310c43807]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:55.085384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfff49ef-efbf-4e3d-80f5-9db00621aaf2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 533109, 'reachable_time': 32440, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181407, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:55.092434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:13:55.092795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6302dd37-5d75-4149-bb95-ce17fcbda896]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:55.093811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "504d40a8-4bf0-4800-a12e-8455b5dbbc9c", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.147", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3c47ad8f-835c-497e-90af-2539f11528b7", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4270, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c2:4e:3d", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1570ccc8-2bc5-4e17-b51a-f21f10c65d8e", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:38Z", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:53Z", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:55.097016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:53Z", Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 13, Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 1, update_id 4c327ea5-9ea7-4827-9872-24a829b4b987. Wait time elapsed: 1.072 Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 1, update_id 4c327ea5-9ea7-4827-9872-24a829b4b987. Wait time elapsed: 1.072 Jul 01 10:13:55.109329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:13:57.707858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.609s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:13:57.708464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:13:57.709294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:13:57.709647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:57.709928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:57.711197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[225c65b1-228e-450d-8199-ff9a1853f026]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:57.724133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30d0274e-0183-4a6e-b28d-119a9f768607]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:57.745681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f0a04fe-86b6-4399-99da-7b26a849938f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:57.776862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[929913d6-fd29-40a0-b15a-e91d8686b3a7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181460, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:57.802342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f78ec2a3-ae8a-42a8-b37a-f54d4c29c937]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:57.824752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed51b52e-5b63-4020-b1b1-62272024320d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:57.853954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0cc0206-7d3d-4335-9655-6d7e48ce9c79]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:57.876736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6badd163-b84c-4e44-af0d-51a00b8c1e00]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:57.877385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:57.877720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:57.878170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:13:57.878600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:13:57.899930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:13:57.901422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f36b4e2-93d2-4d42-b919-8f6386fdc32c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:57.904267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1740969-a330-4faf-84a1-8816615c5c6f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:57.905198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:57.945577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:57.946261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:57.946615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:57.947716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:57.948389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:57.998191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6fcf9a6a-f4c0-47dd-8e33-d660ec2ab1ea]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:58.055492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2ee773cb-412d-48e1-b705-20867259f81e]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:13:58.055492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:13:58.055492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:13:58.055492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:13:58.055492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:13:58.055492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:13:58.055492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:13:58.055492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:13:58.055492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:13:58.055492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:13:58.055492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:13:58.055492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:13:58.055492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:58.057027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ee773cb-412d-48e1-b705-20867259f81e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:58.057275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:58.059935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:58.065273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4a8f092-bd62-4262-af6e-560765c664d5]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:58.066045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:58.066359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:58.066769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:13:58.067109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:58.068266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8aeb81c8-1444-490d-b94b-4bd48b22e23e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:58.070114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4905dc13-9e5d-4ad6-b9a1-cf6c4fd9c509]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:58.071386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27403ac4-469f-4d0c-8f17-6a5888c46def]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:58.104662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d70f7cc3-deb7-42cf-9b7a-c7a79396f4a4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 533404, 'reachable_time': 20712, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181480, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b364f6c4-5c0e-4bb0-81e2-a2928c0bb7c0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:58.109363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "504d40a8-4bf0-4800-a12e-8455b5dbbc9c", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.147", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3c47ad8f-835c-497e-90af-2539f11528b7", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4270, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c2:4e:3d", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1570ccc8-2bc5-4e17-b51a-f21f10c65d8e", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:38Z", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:53Z", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:56Z", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:58.116888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id efa4df34-2a81-438d-b96e-030f25c81e10. Wait time elapsed: 21.399 Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id efa4df34-2a81-438d-b96e-030f25c81e10. Wait time elapsed: 21.399 Jul 01 10:13:58.134035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:13:59.082722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.973s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:13:59.084148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:13:59.086533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:13:59.088292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:59.089064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:59.090283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b4cb9ad-85f1-47a6-b4e5-3582ad90033d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.109319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0326caea-5ab9-47d0-b172-36f0cd1cf78d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.133169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9df79040-6dfd-49bc-b362-f19542bcd584]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.160518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f8f8d3a-fd23-446c-a377-8dcf9eb9e53d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181488, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.185046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[386ae7eb-d20e-478e-8e7f-b92a60672714]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.209544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbb78b53-e830-42fe-839f-d34935c862f1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.234849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ca1a4d7-b1cc-47ab-a6d9-273e50892b89]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.256351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a325bf19-efa0-4981-a10d-0bcfa56422f8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.256934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:13:59.257214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:13:59.257503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:13:59.257801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:13:59.274351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:13:59.274641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2990466-b161-4d88-9fc3-3b0b819aa2f4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.278244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3920f1fd-0372-4502-800e-ef3dcc40bd34]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.279171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:59.315540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:59.316946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:59.317623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:59.318319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:59.318913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:13:59.362947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0da84a7-7f21-41eb-84d8-59525b6b6fc6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.402040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e08072a6-a53f-4883-8846-3cc867b1f09b]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:13:59.402040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:13:59.402040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:13:59.402040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:13:59.402040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:13:59.402040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:13:59.402040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:13:59.402040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:13:59.402040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:13:59.402040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:13:59.402040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:13:59.402040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:13:59.402040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:59.403043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e08072a6-a53f-4883-8846-3cc867b1f09b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:59.403439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:59.405161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:59.408813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[125475a2-33e4-4ff2-9139-7d9c113e6597]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.408813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:59.408813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:59.408813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:13:59.408813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:13:59.409884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7df5fd31-e8b7-405b-b3fe-aa7c95918f2f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.412358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d7a2224-e145-45fc-b00e-e784c0c57bae]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.413540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2806821b-ae47-462e-959c-9717e938c708]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.443673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f84d0d3-39b6-4102-8143-59b8194d4c70]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 533542, 'reachable_time': 23876, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181507, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.451721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:13:59.452043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64e332ed-a2e8-4d59-a989-59380473b6d9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:13:59.453330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "504d40a8-4bf0-4800-a12e-8455b5dbbc9c", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.147", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3c47ad8f-835c-497e-90af-2539f11528b7", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4270, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c2:4e:3d", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1570ccc8-2bc5-4e17-b51a-f21f10c65d8e", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:38Z", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:53Z", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:13:59.456353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:58Z", Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4fef0a19-d1df-4942-a576-b40c6e1079bf. Wait time elapsed: 24.650 Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4fef0a19-d1df-4942-a576-b40c6e1079bf. Wait time elapsed: 24.650 Jul 01 10:13:59.469538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:14:02.477760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.023s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:14:02.478823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:14:02.482179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:14:02.483054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:02.483424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:02.484814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76d95ed7-8850-4fce-9ab9-92b1462b4db1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.501362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b64beef6-519c-47c0-aa3b-df4b62e660f7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.522730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[afeea6d9-97e2-403b-95b4-177aca24f8c1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.552685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef600fd9-b600-4a4e-959a-eb345c14a6f2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181557, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.574213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b8c0ad1-4abe-4ba7-99c9-93a7498a5cdd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.596711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83c7695b-7a6a-49f0-a235-8376f15be6ce]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.609684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94ade742-fbca-4875-aff3-f15c6155f1e4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.644538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f7a94c8-433b-412c-aa30-187d8c8565b9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.645387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:02.645890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:02.646336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:14:02.646799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:14:02.680745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:14:02.681048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a46a2880-1f5e-4fed-874c-93231059fa34]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.683977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc6a4748-4ca7-4281-92b3-11ad4469e77d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.684922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:02.718805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:02.719583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:02.720197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:02.720788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:02.721474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:02.772107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8f65a05-9b62-4433-a3ba-99c8c6175c5b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.805602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[98b79933-fad6-4276-b6e1-0e4f60711586]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:14:02.805602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:14:02.805602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:14:02.805602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:14:02.805602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:14:02.805602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:14:02.805602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:14:02.805602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:14:02.805602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:14:02.805602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:14:02.805602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:14:02.805602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:14:02.805602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:02.806785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98b79933-fad6-4276-b6e1-0e4f60711586]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:02.807644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:02.809387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:02.811500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9341cf0b-9826-4aae-82e7-8f0afc31eaf7]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.811500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:02.811500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:02.811500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:14:02.811836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:02.812896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7fe7a984-4e69-4e44-a46b-61be35f887e6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.813913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3711cc6a-7d7c-4867-8735-4a26d5c6d99d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.815146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a44a9df-8900-40ed-80f1-e34bb3600dd7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.839283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b0ef5a0-9677-4352-b4b8-c5dc655d5d2f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 533881, 'reachable_time': 29376, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181577, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f989d285-1047-4653-a6c6-70dc8098eaa5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:02.843609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "504d40a8-4bf0-4800-a12e-8455b5dbbc9c", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.147", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3c47ad8f-835c-497e-90af-2539f11528b7", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4270, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c2:4e:3d", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1570ccc8-2bc5-4e17-b51a-f21f10c65d8e", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:38Z", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:53Z", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:14:00Z", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:02.848278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3 Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4c327ea5-9ea7-4827-9872-24a829b4b987. Wait time elapsed: 8.822 Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4c327ea5-9ea7-4827-9872-24a829b4b987. Wait time elapsed: 8.822 Jul 01 10:14:02.872259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:14:03.573936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.728s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:14:03.574556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:14:03.575572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:14:03.575902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:03.576187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:03.577502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b01f2bf-60de-4356-8670-90a36e6a53db]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.593326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e5bd5df-46ce-48ba-a059-29d484d3e53d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.613891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63b69e06-c5c6-4684-9d59-fce79ea22e69]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.641023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[540f3d99-b2ce-4a4a-93d7-480a4cb96653]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181580, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.663683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19a7abca-ee67-488a-a0f0-52d383da9f0b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.685026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45811377-f857-485a-a622-85aec7f44643]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.705846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5dec4516-f64f-4a39-986d-18f7b5bd9850]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.727347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56fbbddc-6f7f-4d7c-a8d8-6d23a5ac6e1d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.727970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:03.728268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:03.728595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:14:03.729060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:14:03.743985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:14:03.744385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e133d79e-81db-457a-9d07-f631fa427244]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.747148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61e69062-0628-4bea-a45a-f0e994e2f744]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.748188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:03.783238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:03.784015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:03.784483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:03.785521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:03.785521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:03.820345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41b38b6d-8b62-48dd-b611-cd42fb06316e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.865632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d26cd93f-9447-4ec2-b851-86090c7ee88d]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:14:03.865632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:14:03.865632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:14:03.865632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:14:03.865632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:14:03.865632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:14:03.865632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:14:03.865632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:14:03.865632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:14:03.865632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:14:03.865632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:14:03.865632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:14:03.865632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:03.866766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d26cd93f-9447-4ec2-b851-86090c7ee88d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:03.867687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:03.869931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:03.872126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2dff4c2d-8d6c-4879-830c-a704826ad55b]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.872126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:03.872126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:03.872126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:14:03.872126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:03.872793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a219c4bd-aadd-4302-a2a5-31838313b2d1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.874011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74faedd9-5fe7-4726-8028-e9c1f54fbba0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.874942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[598a0236-fb16-470c-ab54-a83ab6dd3251]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.893910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5763f7cb-2c17-4530-a1bb-5b1ed4408597]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 533990, 'reachable_time': 26712, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181599, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.898843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:14:03.898843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41acdc02-3a5c-494d-b4c3-f74c5c9c468b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:03.899911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "504d40a8-4bf0-4800-a12e-8455b5dbbc9c", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.147", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3c47ad8f-835c-497e-90af-2539f11528b7", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4270, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c2:4e:3d", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1570ccc8-2bc5-4e17-b51a-f21f10c65d8e", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:38Z", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:53Z", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:03.903594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:14:02Z", Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id efa4df34-2a81-438d-b96e-030f25c81e10. Wait time elapsed: 27.191 Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id efa4df34-2a81-438d-b96e-030f25c81e10. Wait time elapsed: 27.191 Jul 01 10:14:03.916690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:14:06.584693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.682s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:14:06.585342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:14:06.585961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:14:06.586282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:06.586561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:06.588587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a15e06f7-249f-43e8-9b69-f953ff4f6a39]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.602196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b493e37e-8117-4497-ba40-727231b4ef61]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.623329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32ffb240-32bd-4f9d-9386-472632da666d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.652276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ced8a1e1-4b25-4f02-a644-88bbf929d730]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181651, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.673081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[370fa79c-8299-4917-a14b-9929a16675e6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.693881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a58d373b-f6cf-4e56-a9c7-31c1328b09e8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.716414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ce55f20-a752-4477-8bd1-e3733768669a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.737324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9edf035b-a11e-439b-8116-ba11e04cd7af]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.737915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:06.738193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:06.738463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:14:06.738743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:14:06.754367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:14:06.754784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bacde877-6190-4544-a41f-597004093883]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.757349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53e46ee7-253a-4b02-9080-60127640bb54]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.758492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:06.790361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:06.791802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:06.792306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:06.792827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:06.793414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:06.851634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6fecc90-d086-4568-a8ee-84cf89b9917b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.882194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0e1cc712-a94c-4131-8aa7-58311bc64dac]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:14:06.882194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:14:06.882194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:14:06.882194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:14:06.882194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:14:06.882194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:14:06.882194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:14:06.882194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:14:06.882194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:14:06.882194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:14:06.882194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:14:06.882194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:14:06.882194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:06.883691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e1cc712-a94c-4131-8aa7-58311bc64dac]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:06.885176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:06.887051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:06.889197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52dfbf6b-388c-45f5-af3c-a1668320319d]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.889479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:06.889664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:06.890068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:14:06.890587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:06.891674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b39a6b2-17c1-4d5b-80ba-01c56c6facfd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.892748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ec3141e-8019-4b33-ad50-3046c5b92bec]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.893853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27887e17-d268-433e-af4b-092109df42c5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.915332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29ef5010-41f7-4498-b4ea-bd14dcb45952]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 534291, 'reachable_time': 15368, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181669, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.922567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:14:06.922860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e01128d7-4e93-4a16-93e3-5f68d9bf333f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:06.923913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "504d40a8-4bf0-4800-a12e-8455b5dbbc9c", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.147", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3c47ad8f-835c-497e-90af-2539f11528b7", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4270, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c2:4e:3d", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1570ccc8-2bc5-4e17-b51a-f21f10c65d8e", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:38Z", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:53Z", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:06.927494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:14:05Z", Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3 Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4c327ea5-9ea7-4827-9872-24a829b4b987. Wait time elapsed: 12.902 Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4c327ea5-9ea7-4827-9872-24a829b4b987. Wait time elapsed: 12.902 Jul 01 10:14:06.941409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:14:07.787482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.862s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:14:07.788105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:14:07.789240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:14:07.789567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:07.789854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:07.791027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53a2e005-547a-430a-923f-17b775f2bbfe]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:07.802640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9d0a640-42c0-429e-bec9-2f44a19ed255]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:07.824133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a2c649c-45d5-46fd-b756-68af2d7ff587]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:07.850743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b39818d-def0-4dfb-9eff-5d08eda95aa8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181673, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:07.872337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[397b83f0-4ac1-46ab-94ec-a540381bb08f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:07.893181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89a8f6d4-bc28-4dbc-9f57-a1fc01794af1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:07.913600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65ee5c7e-c8f5-4c13-8958-a1464f5e5820]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:07.935090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aefa7076-fc5f-4102-8de9-ff47d4f7fd0f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:07.935696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:07.936045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:07.936311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:14:07.936588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:14:07.951193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:14:07.952076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e8b1910-4016-4e98-8508-ea1150fd270e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:07.954768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14755a1e-0c71-4b1e-8d53-0cf2bfd701dc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:07.956234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:08.009372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:08.010648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:08.011036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:08.011456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:08.011850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:08.061646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd91d6e8-6c22-45eb-b6be-750eadc3518e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:08.095587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9eef284a-2e3f-484a-b544-be34645be935]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:14:08.095587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:14:08.095587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:14:08.095587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:14:08.095587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:14:08.095587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:14:08.095587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:14:08.095587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:14:08.095587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:14:08.095587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:14:08.095587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:14:08.095587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:14:08.095587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:08.096763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9eef284a-2e3f-484a-b544-be34645be935]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:08.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:08.099486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:08.102047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a098f4f9-898a-4113-9580-dc1983711201]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:08.102578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:08.102994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:08.103423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:14:08.103925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:08.104975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a118fd9e-bea8-4132-b056-dbb9f256e64f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:08.106024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6548a042-08e2-4d48-895c-6d081cdfe7cc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:08.107046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d10ae999-6bf1-4bf8-ac49-3f14981d4df2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:08.126364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15436629-c5a5-489e-8c4e-037c282a2a00]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 534411, 'reachable_time': 18120, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181691, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:08.131772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:14:08.131772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5d52397-616f-4507-958f-17b2d5e37631]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:08.133203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "504d40a8-4bf0-4800-a12e-8455b5dbbc9c", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.147", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3c47ad8f-835c-497e-90af-2539f11528b7", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4270, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c2:4e:3d", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1570ccc8-2bc5-4e17-b51a-f21f10c65d8e", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:38Z", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:53Z", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:08.137529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:14:06Z", Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 21, Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4c327ea5-9ea7-4827-9872-24a829b4b987. Wait time elapsed: 14.111 Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4c327ea5-9ea7-4827-9872-24a829b4b987. Wait time elapsed: 14.111 Jul 01 10:14:08.149860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:14:11.170222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.034s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:14:11.170825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:14:11.171434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:14:11.171765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:11.172047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:11.173237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c3e38d1-234e-44af-80c1-98d4662ba2df]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.187602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99cffea6-3f3a-44a1-87cf-fe867f8d07df]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.207840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b87e313-c6ad-4f28-9660-c5df37a80d6a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.238402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e54d1e2-05c5-4d54-bfdb-e3f67fd7c0f0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181741, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.260222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02502461-e31c-4750-bbf6-346173d2f592]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.280769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41076d3c-1e6c-488f-a80f-e68d92b0d5d0]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.300905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2bf46f83-7273-403f-9450-f3a51bf1996a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.322141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2e4a061-0b4e-4601-aa15-9c8d5136e3d8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.322991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:11.323511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:11.323782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:14:11.324068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:14:11.338783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:14:11.339062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[202c9ac4-c30e-4d79-ace5-e3bedb0753d9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.341716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bae41974-0edc-441e-80fa-07993b0395cc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.342642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:11.378530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:11.379453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:11.379946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:11.380984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:11.381606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:11.430118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37fbf986-ab3a-4ca0-b4f8-e545ffb600bf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.484780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f9c40f21-1003-4841-bc2e-08c2db53367e]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:14:11.484780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:14:11.484780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:14:11.484780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:14:11.484780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:14:11.484780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:14:11.484780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:14:11.484780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:14:11.484780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:14:11.484780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:14:11.484780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:14:11.484780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:14:11.484780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:11.485788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9c40f21-1003-4841-bc2e-08c2db53367e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:11.486070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:11.488268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:11.490812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97abd31e-afba-4ab4-8894-0a6fe5a99097]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.490812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:11.490812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:11.490812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:14:11.491207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:11.492174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b1a0301-cd74-4058-b812-e16c5b31439e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.493173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca78ce9b-c8e9-44a8-93f1-e736e2e152ea]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.494145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[519fb9a7-c892-414b-bf0e-2b2c9d93f174]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.513661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9a093bb-dd58-4a4a-a248-b6bb3f76b612]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 534750, 'reachable_time': 15692, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181759, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.519325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:14:11.519670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9441ed4-44c0-4eee-8470-db29d95215ee]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:11.520721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "504d40a8-4bf0-4800-a12e-8455b5dbbc9c", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.147", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3c47ad8f-835c-497e-90af-2539f11528b7", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4270, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c2:4e:3d", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1570ccc8-2bc5-4e17-b51a-f21f10c65d8e", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:38Z", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:53Z", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:11.524080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:14:09Z", Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 24, Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4c327ea5-9ea7-4827-9872-24a829b4b987. Wait time elapsed: 17.500 Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4c327ea5-9ea7-4827-9872-24a829b4b987. Wait time elapsed: 17.500 Jul 01 10:14:11.538720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:14:12.277373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.753s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:14:12.277990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:14:12.278585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:14:12.278904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:12.279184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:12.280323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5057a240-0a82-4c84-8231-95054fe493ff]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.294876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44fddccb-9bde-4a14-b81b-47d8cf8a681d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.315495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ac745ab-8740-47ad-b8f6-ce295e7e59ed]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.346964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[117ffef2-2941-4c1f-a43a-41692c5da82a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181763, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.368884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07e3b8ca-4ff2-49c6-a111-7449853b9293]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.389413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb60ca68-99a4-421b-8889-14c48927e23e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.409681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb6e610d-8529-4c77-bafb-ef341cbd342d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.432677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce48832c-bd9b-41ca-af9d-351d51a9e8e2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.433248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:12.433522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:12.433786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:14:12.434063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:14:12.449175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:14:12.449564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42518d73-904b-4edc-854c-668965b700f1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.452395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fac760c1-412a-4ea1-aba1-77447c2541ca]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.453408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:12.490935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:12.491639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:12.492180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:12.492574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:12.492960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:12.540927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2cb8f9f-2406-4737-8c3a-daf7e51b2ad0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.575345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[afb7d99f-5929-4cd1-af08-339cd7c7e60a]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:14:12.575345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:14:12.575345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:14:12.575345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:14:12.575345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:14:12.575345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:14:12.575345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:14:12.575345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:14:12.575345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:14:12.575345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:14:12.575345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:14:12.575345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:14:12.575345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:12.576507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[afb7d99f-5929-4cd1-af08-339cd7c7e60a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:12.576760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:12.578786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:12.581312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aba67f00-9c79-4004-988d-add25d2f0904]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.581312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:12.581312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:12.581312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:14:12.581312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:12.581775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0d38528-2e27-48a1-a517-5502947437e1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.583020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[346a0cc9-b91a-472b-ae91-1fe307b8cda0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.583976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecc4630f-9b9d-4868-a404-28d822a90f23]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.603307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed6beacf-a41c-4018-a953-2f83b6e1afc5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 534860, 'reachable_time': 22728, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181781, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.611408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:14:12.611408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a28cebb-2a9b-444f-9217-273d13a79581]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:12.612719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "504d40a8-4bf0-4800-a12e-8455b5dbbc9c", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.147", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3c47ad8f-835c-497e-90af-2539f11528b7", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4270, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c2:4e:3d", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1570ccc8-2bc5-4e17-b51a-f21f10c65d8e", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:38Z", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:53Z", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:12.617053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:14:11Z", Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 25, Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4c327ea5-9ea7-4827-9872-24a829b4b987. Wait time elapsed: 18.594 Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 4c327ea5-9ea7-4827-9872-24a829b4b987. Wait time elapsed: 18.595 Jul 01 10:14:12.628196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:14:15.126342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:15.126915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:15.126915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:15.127112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:15.238914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:14:15.626082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.007s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:14:15.626418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:14:15.627437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:14:15.627831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:15.628178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:15.629336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[964cc083-491b-4232-898c-da2cca8f6c46]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.647276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1459f748-958c-4a97-8051-61f089a84657]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.668223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2931b7d8-72fe-4b3a-9311-5f4377aa5c89]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.696406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f243b02-c914-48c4-b21a-d53f22b05d9c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 181831, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.717831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4bd1f8d2-39b6-41d7-919c-8c61db00acc9]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.738557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9089c3cf-7f75-41cb-985d-10bbff969c37]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.760586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00b2e004-eca1-4d84-943a-b299a0ba2d58]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.781190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81c760d0-46b7-49f7-a98a-c7dee317ffe6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.781795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:14:15.782101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:14:15.782362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:14:15.782639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:14:15.798188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:14:15.798487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcd0ae9a-abbb-442c-a953-1c34cf3b92ad]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.801063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1868d50-f751-4f22-984d-60fa7f06ab69]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.802031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:15.845550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:15.846217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:15.846609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:15.846989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:15.847387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:14:15.896485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[498dc257-bb5b-48eb-bff2-4c5d5c748114]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.923110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[75823f6d-edee-463e-bc2b-879c619efa92]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:14:15.923110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:14:15.923110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:14:15.923110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:14:15.923110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:14:15.923110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:14:15.923110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:14:15.923110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:14:15.923110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:14:15.923110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:14:15.923110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:14:15.923110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:14:15.923110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75823f6d-edee-463e-bc2b-879c619efa92]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:15.924363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:15.926416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:15.929015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fcd3d81-c291-4af2-826c-38d64eed1cc0]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.929015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:15.929282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:15.929516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:14:15.929915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:14:15.930983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e147ffd-0446-47e3-98ec-b6af5f332d23]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.932762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b43696e-01d7-45a3-ac62-d7b0f0f73b36]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.933854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42ef4e62-f0f5-4689-b3d6-884edc5eb49a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.953209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b71f83c4-5036-4199-aab1-ba8b02e4a9d8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 535196, 'reachable_time': 19316, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 181849, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.959159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:14:15.959527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b08b7a23-d885-43c2-a6b7-2bdc9bb9acd4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:14:15.960918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "504d40a8-4bf0-4800-a12e-8455b5dbbc9c", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.147", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3c47ad8f-835c-497e-90af-2539f11528b7", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4270, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c2:4e:3d", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "1570ccc8-2bc5-4e17-b51a-f21f10c65d8e", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:38Z", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:53Z", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:14:15.964115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:14:13Z", Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 28, Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:14:15.976522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3 Jul 01 10:14:55.240347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:15:15.122681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:15:15.123422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:15:15.123422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:15:15.123422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:15:35.242182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:16:15.123531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:16:15.124218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:16:15.124218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:16:15.124491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:16:15.243264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:16:55.244913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:17:15.123945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:17:15.124582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:17:15.124582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:17:15.124793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:17:35.246714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:18:15.124258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:18:15.124949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:18:15.124949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:18:15.124949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:18:15.248647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:18:55.251162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:19:15.125215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:19:15.125811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:19:15.125811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:19:15.126220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:19:35.252886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:20:15.125683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:20:15.126401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:20:15.126401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:20:15.126615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:20:15.254715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:20:55.256447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:20:56.641261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-64ba2794-ca5e-4f9c-86f1-3930ebcff62d tempest-ServerActionsTestJSON-621858079 tempest-ServerActionsTestJSON-621858079-project] Got routers updated notification :['f0193d7f-79a0-4dc0-b92f-43b1cb63c572'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:20:56.642500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 1, update_id 61b8373c-edca-4153-a780-00eeac013372. Wait time elapsed: 0.001 Jul 01 10:20:56.645147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 1, update_id 61b8373c-edca-4153-a780-00eeac013372. Wait time elapsed: 0.001 Jul 01 10:20:56.645542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:20:57.617358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.972s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:20:57.617633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:20:57.618606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:20:57.618934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:20:57.619219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:20:57.621071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[551b4dae-88a9-4009-ba73-914d5b3a3cb0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:57.655205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d009612-c972-44af-bf42-9b21e50acda1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:57.685386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37b88003-ebdb-4a0c-9d3d-0d5b8c195a0f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:57.727880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[018e0df9-7b22-4072-bfef-c1b2b303e275]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188213, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:57.756974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35501a83-9d7d-43a0-8983-b0bbb2186e94]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:57.778438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2489990a-cf4d-453e-8e5b-0b9bd0a5b34a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:57.804228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24d5ac66-1f6a-43e4-a9c2-bb2328e732d4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:57.827487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92a68644-fcfc-4c29-881a-78a82970e8ee]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:57.828283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:20:57.828569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:20:57.828842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:20:57.829131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:20:57.845942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:20:57.846154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05b181cf-66fe-4ef7-ad74-56922c5493d7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:57.849266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1e9c380-9d5c-47ab-8220-fb4896459f72]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:57.850341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:20:57.900381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:20:57.902718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:20:57.903709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:20:57.904604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:20:57.905210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:20:57.969189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[342d9f4a-d744-42e7-b361-658f6fd5cb06]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:58.026941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[79779ec6-94eb-41c0-8eeb-6ee97da7c28c]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:20:58.026941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:20:58.026941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:20:58.026941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:20:58.026941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:20:58.026941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:20:58.026941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:20:58.026941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:20:58.026941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:20:58.026941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:20:58.026941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:20:58.026941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:20:58.026941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:20:58.027771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79779ec6-94eb-41c0-8eeb-6ee97da7c28c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:20:58.028134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:20:58.032596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:20:58.038146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5793bc85-6226-4c26-8282-900ffed3bcd7]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:58.039626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:20:58.040116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:20:58.040517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:20:58.041162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:20:58.043094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b6a368c-d989-469c-b8a8-4570fd2d0417]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:58.044633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8e8882c-5be1-4c2c-b13c-3c352ef7b37e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:58.045901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7846837f-c01f-4fce-883d-a6baf917b9de]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:58.082346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e098511-60a8-4312-b3e4-3a7172fcb371]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 575398, 'reachable_time': 21504, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188233, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:58.089532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:20:58.090263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49d3a369-b2d6-448b-b2cc-66d504102cc9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:20:58.091761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:14:19Z", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 31, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:20:58.096416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:20:58.110938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 61b8373c-edca-4153-a780-00eeac013372. Wait time elapsed: 1.466 Jul 01 10:20:58.110938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 2, update_id 61b8373c-edca-4153-a780-00eeac013372. Wait time elapsed: 1.466 Jul 01 10:20:58.110938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:00.051041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-0503a439-d799-44d4-8f69-b5feab0e464c admin admin] Got routers updated notification :['f0193d7f-79a0-4dc0-b92f-43b1cb63c572'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:21:00.452956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.344s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:00.453430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:21:00.457254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (f0193d7f-79a0-4dc0-b92f-43b1cb63c572), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:21:00.457771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:00.458064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:00.459744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a960c34-3cec-4527-a26a-df9b19fc6aa4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.479926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1724656-430c-4964-b290-cabac5fda10d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.502657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[931ac1f4-b508-40a1-8bda-3764ea3a5d6e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.533492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f37c1158-91a3-49ba-9816-c772ba1dcd0f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188244, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.555690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2be113a-1edc-4403-ba3c-d5d71a3b30ad]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.577019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f87209b7-14f3-4069-9835-fdf0fa7ef24a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.610286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c7df7c5-a8b6-46a2-ac62-df4ddbc01b90]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.633977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5aa779b2-727f-4ca4-9400-099b83c99c7e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.634720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:00.635713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:00.636597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:21:00.637085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(2fe1480f-23b4-47c7-b7ae-b8b584a21942) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:21:00.657813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:21:00.658496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c73503d-5d80-4854-bd17-7007c56dc2ba]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.661389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51533ad3-c9a3-4ece-bff0-16cb216d3b7e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.662570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-2fe1480f-23, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:00.711015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-2fe1480f-23, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:00.714471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:00.714471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:00.719643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-2fe1480f-23, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:00.720174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-2fe1480f-23, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '2fe1480f-23b4-47c7-b7ae-b8b584a21942', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:76:74:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:00.771874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aace3072-1135-43f9-87c1-3a00829c9939]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bc4d1682-d103-40df-ab87-109e8058a027]: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:21:00.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:21:00.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:21:00.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:21:00.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:21:00.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:21:00.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:21:00.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:21:00.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:21:00.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:21:00.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:21:00.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:21:00.804870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:21:00.805960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc4d1682-d103-40df-ab87-109e8058a027]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:21:00.806041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:21:00.807637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:00.810171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1728eb4-9e5f-4197-ae99-27cb5c4949ee]: (4, ['qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.811131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:00.811619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:00.811992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:21:00.812458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/f0193d7f-79a0-4dc0-b92f-43b1cb63c572.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:00.813660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8af01f8e-8eec-4039-8ba0-7b21aefb789b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.815126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf06f530-2edb-44b0-bea5-df4f4864daa5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.816259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96c6a219-2552-4b5e-a02d-e74a1d36e5e0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.845004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e70119f8-38a0-4093-b71a-4a8440bf4d42]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 575679, 'reachable_time': 34084, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188262, 'error': None, 'target': 'qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.849563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:21:00.850130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ece91f7e-058f-44d6-afdf-3bd34e399cb3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: f0193d7f-79a0-4dc0-b92f-43b1cb63c572: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-2fe1480f-23 not found in namespace qrouter-f0193d7f-79a0-4dc0-b92f-43b1cb63c572. Jul 01 10:21:00.852504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServerActionsTestJSON-1937797724-router", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220" Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:32Z", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:35Z", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f31e69d-f38a-4748-82eb-082fe6e029b9", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:0b:81:f4", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4259, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.220", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:33Z", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:13:39Z", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "2fe1480f-23b4-47c7-b7ae-b8b584a21942", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "15ff82fe-dd1d-4445-a103-1e2c3c6102b5", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:76:74:88", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "f0193d7f-79a0-4dc0-b92f-43b1cb63c572", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4263, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:13:35Z", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:20:58Z", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 32, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "c10800d50084466ea90e111cf585dbc2", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "34884f4f-90b3-4933-a582-c290af15a3fc", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:21:00.855251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:21:00.867105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 1, update_id 092e9833-8711-4650-b2bc-184771e35bb3. Wait time elapsed: 0.803 Jul 01 10:21:00.867105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 3, priority 1, update_id 092e9833-8711-4650-b2bc-184771e35bb3. Wait time elapsed: 0.806 Jul 01 10:21:00.867105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:00.915597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ec904ea0-5927-4144-81d2-3ce4c36c5726 admin admin] Got router deleted notification for f0193d7f-79a0-4dc0-b92f-43b1cb63c572 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:21:01.050275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.193s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:01.050645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 was not found. Performing router cleanup Jul 01 10:21:01.052146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed57cfd0-97c8-4968-9d98-bb2ac6fe6681]: (4, ['qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:01.053345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:01.053691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:01.054012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, update_id 092e9833-8711-4650-b2bc-184771e35bb3. Time elapsed: 0.199 Jul 01 10:21:01.054359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 1, priority 1, update_id 6a6d4a40-9c8a-4b1d-a9df-e040cc4962d8. Wait time elapsed: 0.138 Jul 01 10:21:01.055127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, action 1, priority 1, update_id 6a6d4a40-9c8a-4b1d-a9df-e040cc4962d8. Wait time elapsed: 0.139 Jul 01 10:21:01.055489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router f0193d7f-79a0-4dc0-b92f-43b1cb63c572 was not found. Performing router cleanup Jul 01 10:21:01.056431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93728f8b-4582-4f21-96fd-6341a489a34f]: (4, ['qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:01.058486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:01.059749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:01.060178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for f0193d7f-79a0-4dc0-b92f-43b1cb63c572, update_id 6a6d4a40-9c8a-4b1d-a9df-e040cc4962d8. Time elapsed: 0.006 Jul 01 10:21:05.768618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8b11bc49-8770-4c6a-b36a-c6916b6e3f58 admin admin] Got routers updated notification :['0a2cfd54-251d-46f3-934d-906d043b085c'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:21:05.770291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 1, update_id b7cb9f64-5bb3-4822-b756-8117d970b73c. Wait time elapsed: 0.001 Jul 01 10:21:05.770509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 1, update_id b7cb9f64-5bb3-4822-b756-8117d970b73c. Wait time elapsed: 0.001 Jul 01 10:21:05.773098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:07.772982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-a543e380-2a05-44fc-b688-3caa50071741 admin admin] Got routers updated notification :['0a2cfd54-251d-46f3-934d-906d043b085c'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:21:09.369060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.596s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:09.369689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a2cfd54-251d-46f3-934d-906d043b085c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:21:09.373937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a2cfd54-251d-46f3-934d-906d043b085c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:21:09.374292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:09.374583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:09.375946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7ddc2f1-829c-4987-897e-a1906ec2b0bb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.389854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[523d6c9d-2dd3-4aa7-a538-c0548b772538]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.411817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[232ca06e-8754-4344-8d65-0eed0b676473]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.445363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63420c9f-ce1c-4240-a481-7ce23925ad91]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188396, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.469911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3303e3cf-72b4-4560-aace-6729026c679e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.491808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93434550-8643-4496-9e81-dfea9c504f6d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.517338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de47c87f-b3cc-4433-ba0d-83df17dda9b3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.538808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7367b7af-78c8-457d-8d46-d54a106af541]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.539378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:09.539677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:09.539956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:21:09.540233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(90c12a68-55c9-4da5-b906-9be90ef49231) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:21:09.554921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:21:09.555210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2169cbb8-2207-4a1a-a3e9-a7aeb63a8616]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.558004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f354739f-ab59-432a-9b4c-6436c63186d0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.559073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-90c12a68-55, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:09.559431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:21:09.560206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-90c12a68-55, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:09.560902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:09.561297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:09.561717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:09.562127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-90c12a68-55, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '90c12a68-55c9-4da5-b906-9be90ef49231', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:27:e8:c2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:09.599759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9eca7df-0162-4f0c-ba70-252b353f0de7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bbb08103-0a41-4ab6-9622-70f27b23834c]: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:09.632048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbb08103-0a41-4ab6-9622-70f27b23834c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:09.635973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:09.639172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:09.639172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:09.639172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:09.639172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:09.639172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:09.639172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:09.639172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:09.639172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:09.641468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a0231e7-c31f-46ec-964c-e3ba34850aea]: (4, ['qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.644645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:09.645173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:09.645582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:21:09.646007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:09.647447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c3bc655-588b-44e7-9737-243913069100]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.648842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71bd3da2-d70b-47d5-80fb-3c9e0acbd39a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.650158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[577ca3c6-288e-422e-a88d-d6e317890fb1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.670334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a6f8851-41f3-4d6e-b1a9-838ea9ad73b6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 576570, 'reachable_time': 22636, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188414, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.674183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:21:09.675647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c278d986-7955-411c-b715-e6764b216605]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:09.676830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersNegativeTestJSON-718025888-router", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63" Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:47:d3:7f", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4700, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "90c12a68-55c9-4da5-b906-9be90ef49231", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b7c4afb-9649-49f5-b98f-7e26bf2b255d", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:27:e8:c2", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4704, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:06Z", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 2, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 1, update_id 5cbb4b48-75e2-4895-8ac8-e329641ad12c. Wait time elapsed: 1.905 Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 1, update_id 5cbb4b48-75e2-4895-8ac8-e329641ad12c. Wait time elapsed: 1.905 Jul 01 10:21:09.679383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:11.082311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.404s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:11.082879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a2cfd54-251d-46f3-934d-906d043b085c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:21:11.083546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a2cfd54-251d-46f3-934d-906d043b085c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:21:11.084778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:11.085064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:11.086650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec3c9803-eaa7-4180-897f-db01608425b9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.103068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[970fb537-2bb0-4b3f-8ad2-85501aef8cc6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.134747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8082a42-abfc-4b02-b4a5-1d5d9427640e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.164310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[456c39b8-dd72-4a9c-bc05-cd50820f1775]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188418, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.189669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd705806-9faa-4c75-ab6d-5ec2e0f8c134]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.220003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c495f0c0-a79b-475f-95ed-1737adfcf0b2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.241924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59b265dc-bb5f-48e4-9c1d-38ce22e17823]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.266754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c6525d1-27df-41b4-bc54-5ce3cffa2b62]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.267557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:11.267889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:11.268219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:21:11.268563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(90c12a68-55c9-4da5-b906-9be90ef49231) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:21:11.283192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:21:11.283663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75caa73c-2fd5-493b-a4e4-659ad4b3c295]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.286292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa3d3bd2-abf1-463e-945f-f1e764b52c31]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.287223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-90c12a68-55, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:11.326947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-90c12a68-55, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:11.327853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:11.328346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:11.328827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:11.329459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-90c12a68-55, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '90c12a68-55c9-4da5-b906-9be90ef49231', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:27:e8:c2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:11.389259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d365eca-5c21-4992-bc01-915282d9c97e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.423107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[27df332a-3259-4dde-bbec-db226bad6f6b]: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:21:11.423107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:21:11.423107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:21:11.423107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:21:11.423107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:21:11.423107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:21:11.423107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:21:11.423107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:21:11.423107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:21:11.423107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:21:11.423107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:21:11.423107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:21:11.423107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:11.423965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27df332a-3259-4dde-bbec-db226bad6f6b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:11.424308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:11.425785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:11.437122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6565e7b3-f444-499d-801b-9cda7eb9918b]: (4, ['qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.437936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:11.438255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:11.438949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:21:11.439307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:11.440421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9645cbfd-62c8-4ad5-abbe-66aafc3d442c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.441474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f94e4e75-21bf-48e1-a9b1-b9a69cd86fc4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.442453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20024a92-608c-476a-bdf3-6d07e64ffd90]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.464844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84b9a2bc-c445-413a-b7f8-b0cbbcc73196]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 576742, 'reachable_time': 15236, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188438, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.472760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:21:11.473533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f962a95-8e72-4440-87fd-be7c30633786]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:11.474557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersNegativeTestJSON-718025888-router", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63" Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:47:d3:7f", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4700, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "90c12a68-55c9-4da5-b906-9be90ef49231", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b7c4afb-9649-49f5-b98f-7e26bf2b255d", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:27:e8:c2", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4704, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:06Z", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:10Z", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:21:11.477071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id b7cb9f64-5bb3-4822-b756-8117d970b73c. Wait time elapsed: 5.706 Jul 01 10:21:11.488374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id b7cb9f64-5bb3-4822-b756-8117d970b73c. Wait time elapsed: 5.706 Jul 01 10:21:11.488374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:13.695937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.220s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:13.696566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a2cfd54-251d-46f3-934d-906d043b085c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:21:13.697115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a2cfd54-251d-46f3-934d-906d043b085c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:21:13.698722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:13.698722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:13.699120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a040260-f164-48b6-8072-1ebf54755a60]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.712475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[935f2a30-a782-4e87-8558-c96e9811d34e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.735778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f62dfb2-5485-43b1-9d1e-0e044e14ef20]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.764685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e354c67a-44b6-413b-813e-f482fa3921e9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188487, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.780855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47063fc5-59ad-4db3-937a-d06324d77952]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.802345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c5ee413-9678-4ff1-bef1-4959d62876ab]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.822656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d295e05d-dd83-40db-a6e1-b6b633d5e141]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.842517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[931a77d1-0ea7-4ec8-969d-6d7deb478fb6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.843049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:13.843316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:13.843617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:21:13.843891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(90c12a68-55c9-4da5-b906-9be90ef49231) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:21:13.858874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:21:13.859159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[509afc5e-b7d1-4614-b19a-bb4344a45a14]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.861977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69ea815f-2dfe-4fc6-94fa-1099ee3ed7ff]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.862986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-90c12a68-55, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:13.893427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-90c12a68-55, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:13.894081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:13.894514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:13.894892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:13.895294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-90c12a68-55, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '90c12a68-55c9-4da5-b906-9be90ef49231', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:27:e8:c2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:13.941675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2b6cff7-ffe3-4655-868a-eaaa2984d607]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.983344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[45932253-d983-4586-95f9-21c9281ad8a9]: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:21:13.983344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:21:13.983344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:21:13.983344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:21:13.983344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:21:13.983344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:21:13.983344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:21:13.983344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:21:13.983344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:21:13.983344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:21:13.983344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:21:13.983344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:21:13.983344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:13.984330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45932253-d983-4586-95f9-21c9281ad8a9]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:13.984610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:13.986040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:13.988328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33366938-78c5-4ea3-8c6e-159cc0ac2b19]: (4, ['qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.988884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:13.989259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:13.989519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:21:13.989923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:13.990968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb258aa1-eee9-4677-85d6-f8f10e143597]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.992024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4acae21-899c-4f69-9df3-f2c08fea8c06]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:13.993008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4c3fe10-0c42-4c12-8e1e-81e27cc3d271]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:14.012270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c092b1c-400c-4b4e-b7c8-e5e249711ceb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 577002, 'reachable_time': 26064, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188505, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:14.017831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:21:14.018270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d80842d3-b7f4-4310-a900-e9932291f80f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:14.019293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersNegativeTestJSON-718025888-router", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63" Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:47:d3:7f", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4700, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "90c12a68-55c9-4da5-b906-9be90ef49231", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b7c4afb-9649-49f5-b98f-7e26bf2b255d", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:27:e8:c2", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4704, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:06Z", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:10Z", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:21:14.022109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id 5cbb4b48-75e2-4895-8ac8-e329641ad12c. Wait time elapsed: 6.247 Jul 01 10:21:14.040880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id 5cbb4b48-75e2-4895-8ac8-e329641ad12c. Wait time elapsed: 6.247 Jul 01 10:21:14.040880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:14.733069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.712s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:14.733705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a2cfd54-251d-46f3-934d-906d043b085c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:21:14.734246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a2cfd54-251d-46f3-934d-906d043b085c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:21:14.734569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:14.734849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:14.736328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[906351f8-839a-4504-8613-909466b6e546]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:14.888371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de177689-be07-484c-89e5-ce7b052a19d8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:14.910073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86a7b06c-2473-43cb-9da4-484ead9d1163]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:14.940009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f516f00d-ec8a-4c47-a240-ae79a316e3cf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188508, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:14.960770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9e02b7a-7aa4-4066-98b4-930ad359fb7d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:14.981396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ef3e39e-1533-4cb6-a4ce-dd9bca9583ff]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:15.001620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1f69f4a-ae44-40bd-9ff7-34ebbf63855d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:15.022929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9832b194-8159-4869-8d35-69976d81afd2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:15.023628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:15.024501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:15.024872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:21:15.025214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(90c12a68-55c9-4da5-b906-9be90ef49231) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:21:15.043172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:21:15.043569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a259a126-912f-4196-9073-fd37781d569e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:15.048693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d57b80f-2e88-4ecb-8d05-9adf73c39f00]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:15.051593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-90c12a68-55, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:15.089982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-90c12a68-55, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:15.090730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:15.091185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:15.091691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:15.092134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-90c12a68-55, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '90c12a68-55c9-4da5-b906-9be90ef49231', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:27:e8:c2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:15.155033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.004s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:15.155278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:15.160342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:15.160342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:15.172794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[209986c8-12b8-4fed-809a-579f932774d9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:15.215505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3a076bdc-aea1-4c59-8376-98b6d452a5fd]: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:21:15.215505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:21:15.215505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:21:15.215505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:21:15.215505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:21:15.215505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:21:15.215505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:21:15.215505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:21:15.215505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:21:15.215505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:21:15.215505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:21:15.215505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:21:15.215505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:15.216762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a076bdc-aea1-4c59-8376-98b6d452a5fd]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:15.217715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:15.219179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:15.222069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cdfd30f-ef08-4f40-9ca6-55c5b40cacb3]: (4, ['qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:15.222923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:15.223898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:15.224334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:21:15.224671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:15.226947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0db489fc-f50a-48a8-a45f-64cda23bfc4a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:15.228051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aab3960e-31c3-40a0-a176-140fdbe799bd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:15.229377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa7b9d5c-0e09-4864-8233-9fb995262bdf]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:15.256172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f3c5a56-a3d7-47d9-96c6-931e35f305b1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 577120, 'reachable_time': 35952, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188527, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:15.268782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:21:15.270478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8d9f507-7eda-4e29-b289-e55b884cdf85]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:15.271736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersNegativeTestJSON-718025888-router", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63" Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:47:d3:7f", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4700, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "90c12a68-55c9-4da5-b906-9be90ef49231", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b7c4afb-9649-49f5-b98f-7e26bf2b255d", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:27:e8:c2", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4704, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:06Z", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:13Z", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:21:15.274056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id b7cb9f64-5bb3-4822-b756-8117d970b73c. Wait time elapsed: 9.503 Jul 01 10:21:15.303292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id b7cb9f64-5bb3-4822-b756-8117d970b73c. Wait time elapsed: 9.503 Jul 01 10:21:15.303292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:18.253564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.975s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:18.254561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a2cfd54-251d-46f3-934d-906d043b085c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:21:18.255505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a2cfd54-251d-46f3-934d-906d043b085c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:21:18.255980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:18.256250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:18.257814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e37167a9-6d6a-4259-b160-f8c86a46d18e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.282699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa305ff9-3bbe-4dd3-a41d-b1c60ecc777c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.311571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4215ba1-3de5-4813-b75b-4af7412c75db]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.348211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca505bcb-05a4-4425-bb61-d194badf0bf7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188581, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.373008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9442cae8-74a2-47b0-b392-f4d0324a07f0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.393162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f368c06-5a52-430b-b94d-3505a347866e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.405128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80e9f5ba-434d-4cfe-9fb6-7c1c4083b0bf]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.424773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b067cb75-2af8-4ebd-bab7-b04cbea6be71]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.425344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:18.425622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:18.425891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:21:18.426173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(90c12a68-55c9-4da5-b906-9be90ef49231) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:21:18.442439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:21:18.442903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e1ab6a0-4fb7-42e4-95ee-fe5a08b4c29c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.446955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20d75026-6796-40a7-b536-f47e8d9f1b5a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.449933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-90c12a68-55, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:18.485521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-90c12a68-55, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:18.486271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:18.486733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:18.487190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:18.487729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-90c12a68-55, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '90c12a68-55c9-4da5-b906-9be90ef49231', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:27:e8:c2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:18.550552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91fddb92-155c-4753-9f32-f588ee254da5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.591162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[86683cba-46c3-4384-adf4-0a822d9903f1]: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:21:18.591162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:21:18.591162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:21:18.591162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:21:18.591162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:21:18.591162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:21:18.591162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:21:18.591162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:21:18.591162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:21:18.591162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:21:18.591162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:21:18.591162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:21:18.591162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:18.592192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86683cba-46c3-4384-adf4-0a822d9903f1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:18.592473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:18.593884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:18.595665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1be8a59-03f7-42da-9600-6c6a484aa8e6]: (4, ['qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.596477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:18.596794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:18.597046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:21:18.597380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:18.598350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71978eec-b5b0-4887-9889-42b8f02e87c7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.599623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d5d0fe9-be3b-4359-9227-ccbb1b148911]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.600767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[684dc5ef-bc05-41b4-9619-e0b46835654a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.621958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea6804b0-7445-489f-9b5b-8c73496b7534]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 577460, 'reachable_time': 18220, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188599, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.627688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:21:18.627949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efcbe664-cac9-4f3c-8d48-1e6ff15f1395]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:18.629264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersNegativeTestJSON-718025888-router", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63" Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:47:d3:7f", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4700, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "90c12a68-55c9-4da5-b906-9be90ef49231", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b7c4afb-9649-49f5-b98f-7e26bf2b255d", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:27:e8:c2", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4704, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:06Z", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:16Z", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:21:18.631937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id 5cbb4b48-75e2-4895-8ac8-e329641ad12c. Wait time elapsed: 10.857 Jul 01 10:21:18.648129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id 5cbb4b48-75e2-4895-8ac8-e329641ad12c. Wait time elapsed: 10.857 Jul 01 10:21:18.648129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:19.397760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.767s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:19.398816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a2cfd54-251d-46f3-934d-906d043b085c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:21:19.399772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a2cfd54-251d-46f3-934d-906d043b085c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:21:19.400309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:19.400602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:19.402330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b2ea185-7361-44e7-b952-a9a9981a6b70]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.420988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d6c29a0-511f-46e8-a66b-a9a6cfd49d0e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.497973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd48390f-6dd7-4ca7-a95d-97f261ca9b36]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.553548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62cba489-c642-440c-878b-c5115f33930d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188602, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.582658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[253ad9b1-7780-4cab-99d0-edae934a1bcc]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.644499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf60ea79-86b7-4da0-8171-118874ba308f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.669197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b3ab3fc-8bb5-4e3a-8a5c-c21dadc2db97]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.708345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cda3956a-294e-4e86-ae36-74a054390fb9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.709817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:19.710181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:19.710535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:21:19.710872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(90c12a68-55c9-4da5-b906-9be90ef49231) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:21:19.732656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:21:19.733217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41aff506-dc39-4f3d-9766-0228138e9913]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.736456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e8902e5-2a2b-417a-9832-5324d47578b8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.737658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-90c12a68-55, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:19.775149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-90c12a68-55, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:19.777343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:19.778032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:19.778581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:19.778971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-90c12a68-55, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '90c12a68-55c9-4da5-b906-9be90ef49231', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:27:e8:c2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:19.842922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02132638-6d05-482b-b19e-b63c192fab6d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.885008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3fb00b0c-cc16-494b-aa41-264a2c60e9d1]: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:21:19.885008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:21:19.885008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:21:19.885008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:21:19.885008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:21:19.885008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:21:19.885008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:21:19.885008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:21:19.885008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:21:19.885008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:21:19.885008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:21:19.885008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:21:19.885008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:19.885856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fb00b0c-cc16-494b-aa41-264a2c60e9d1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:19.886336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:19.887731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:19.889424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47066da2-2430-40fb-849e-5fd2c8475533]: (4, ['qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.889424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:19.889601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:19.890312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:21:19.890312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:19.891067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd0fdfb2-ba04-4f6c-9d29-f7a7470b909d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.894351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61ff707e-24a0-4f0d-a85c-adddffe0336b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.895608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bf950c4-befd-4168-bf76-d9ac9cf2a9d9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.927414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7213cdc-e4ee-4b8d-94d1-77390e9f269e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 577581, 'reachable_time': 42472, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188654, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.937818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:21:19.938078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62753b77-c3ff-4dba-9ab9-4e5d835af89f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:19.939116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersNegativeTestJSON-718025888-router", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63" Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:47:d3:7f", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4700, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "90c12a68-55c9-4da5-b906-9be90ef49231", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b7c4afb-9649-49f5-b98f-7e26bf2b255d", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:27:e8:c2", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4704, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:06Z", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:18Z", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:21:19.941482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id b7cb9f64-5bb3-4822-b756-8117d970b73c. Wait time elapsed: 14.171 Jul 01 10:21:19.956627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id b7cb9f64-5bb3-4822-b756-8117d970b73c. Wait time elapsed: 14.171 Jul 01 10:21:19.956627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:22.868051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.927s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:22.870311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a2cfd54-251d-46f3-934d-906d043b085c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:21:22.871813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a2cfd54-251d-46f3-934d-906d043b085c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:21:22.872222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:22.872503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:22.873660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b472e8d-aa2d-46a6-b523-a7ac02cac624]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:22.891952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1beb71d8-c918-4c92-ae01-c80bc7aa953c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:22.912898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b7975c3-66f6-420b-b8ea-cae207fcce27]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:22.940667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65c41a85-4e1b-434a-8d43-f413fa0c5feb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188706, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:22.961713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e092a3ed-335d-4f3d-98df-7f64e169da38]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:22.982652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbf0d3aa-0914-4724-a092-99e1f3d9f11c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.003561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63d7bdbd-a6fe-45b4-b813-82ba9cb70aac]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.028192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[641ea2f3-a637-4f6b-8900-107a204477e8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.028768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:23.029040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:23.029302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:21:23.029572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(90c12a68-55c9-4da5-b906-9be90ef49231) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:21:23.049832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:21:23.050379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[085e4134-432f-43fc-b3a9-bb176461f6ec]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.053147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6efc829-188d-44a5-a963-e90a5339d9c7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.054425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-90c12a68-55, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:23.086276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-90c12a68-55, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:23.086936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:23.087731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:23.087731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:23.088812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-90c12a68-55, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '90c12a68-55c9-4da5-b906-9be90ef49231', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:27:e8:c2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:23.134530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76a07a4d-2a12-4084-8e5d-42d9be48c609]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.179171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ef2486eb-9d42-4a5d-b3e5-1f951838e044]: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:21:23.179171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:21:23.179171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:21:23.179171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:21:23.179171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:21:23.179171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:21:23.179171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:21:23.179171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:21:23.179171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:21:23.179171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:21:23.179171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:21:23.179171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:21:23.179171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:23.180023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef2486eb-9d42-4a5d-b3e5-1f951838e044]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:23.180528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:23.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:23.185929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b18fc39-5379-4b86-9591-af395db2ed28]: (4, ['qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.186827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:23.187484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:23.187714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:21:23.188144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:23.189268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[047addac-6b9b-4312-94dc-7a1b367fc88f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.190462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22b11302-c235-4ef6-883d-a7e172f60932]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.192046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6915c5e8-0595-403e-ad62-eaae8b3661c4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.214759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[911f6053-f227-421c-b21f-ff757443b3ff]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 577920, 'reachable_time': 28808, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188724, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82250543-40c6-4e85-946f-9a8adc1f5fa4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersNegativeTestJSON-718025888-router", Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63" Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:23.224031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:47:d3:7f", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4700, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "90c12a68-55c9-4da5-b906-9be90ef49231", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b7c4afb-9649-49f5-b98f-7e26bf2b255d", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:27:e8:c2", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4704, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:06Z", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:20Z", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id 5cbb4b48-75e2-4895-8ac8-e329641ad12c. Wait time elapsed: 15.448 Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id 5cbb4b48-75e2-4895-8ac8-e329641ad12c. Wait time elapsed: 15.448 Jul 01 10:21:23.228462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:23.942235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.720s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:23.942813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a2cfd54-251d-46f3-934d-906d043b085c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:21:23.943570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a2cfd54-251d-46f3-934d-906d043b085c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:21:23.943891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:23.944167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:23.945484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79b10f73-6a78-4a8d-bf59-19b658e8d75e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.962032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f28ef976-5d86-4f74-909b-65b7b5962ff4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:23.982823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b93f7e0-2f54-4c95-a148-2c52ceb7893a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.011305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[309918ec-1fb5-4807-bae8-91e2b9b5836a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188727, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.031772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7780088-9404-47c3-9cab-159c58f70f45]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.051668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa546670-d9fe-41f9-9524-b7a8aa65eba2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.072679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3aaad112-dfd3-41d5-b529-5e3946b79d94]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.093029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3a50946-f44c-497b-8e52-827ff20f78ed]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.093574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:24.093918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:24.094278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:21:24.094622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(90c12a68-55c9-4da5-b906-9be90ef49231) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:21:24.109946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:21:24.110312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[897a1c00-2636-47b0-84a2-ece987dfc475]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.113000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4158eb69-11e9-4a0b-b674-77f1b0cd7ed0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.114065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-90c12a68-55, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:24.149612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-90c12a68-55, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:24.151568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:24.152271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:24.152954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:24.154025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-90c12a68-55, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '90c12a68-55c9-4da5-b906-9be90ef49231', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:27:e8:c2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:24.203677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d214c5c-3e1c-437c-b6bb-ad4f23de1c43]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.281076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4c62cbc7-3111-4c0d-bc1e-bbef5bcd3517]: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:21:24.281076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:21:24.281076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:21:24.281076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:21:24.281076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:21:24.281076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:21:24.281076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:21:24.281076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:21:24.281076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:21:24.281076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:21:24.281076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:21:24.281076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:21:24.281076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:24.282288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c62cbc7-3111-4c0d-bc1e-bbef5bcd3517]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:24.283508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:24.286669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:24.292431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d5f3701-e80c-4a0f-acac-12979316f0e1]: (4, ['qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.293601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:24.294080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:24.294470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:21:24.294939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:24.296438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[605c1c37-4894-4ca8-ac97-469e207c0de0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.297827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0d2dbdc-5c4b-4c2c-8f54-7323f21103a0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.300291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f549ce63-7793-4cb3-ae7b-13fb2f69b01b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.341702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2a039f5-e129-4b9b-9637-4d4795541fb7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 578027, 'reachable_time': 27924, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188746, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.355245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:21:24.355574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[273f76ae-9660-4d49-8cff-9b5ffd638b8c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:24.356628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersNegativeTestJSON-718025888-router", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63" Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:47:d3:7f", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4700, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "90c12a68-55c9-4da5-b906-9be90ef49231", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b7c4afb-9649-49f5-b98f-7e26bf2b255d", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:27:e8:c2", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4704, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:06Z", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:21Z", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:21:24.361040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:21:24.373634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id b7cb9f64-5bb3-4822-b756-8117d970b73c. Wait time elapsed: 18.592 Jul 01 10:21:24.373634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id b7cb9f64-5bb3-4822-b756-8117d970b73c. Wait time elapsed: 18.592 Jul 01 10:21:24.373634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:27.117928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.756s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:27.118542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a2cfd54-251d-46f3-934d-906d043b085c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:21:27.119404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a2cfd54-251d-46f3-934d-906d043b085c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:21:27.119760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:27.120049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:27.124966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[783b7b2a-fc35-4b49-9908-d07935b076b3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.136583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5263b024-b24d-4051-b62a-263e1e288958]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.160499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db9fce17-5110-4faa-bc01-66b4ab730613]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.192560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1315104-f225-42cb-a218-8fc873472fbf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188755, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.216975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bb5591a-3955-4bbf-ab54-11e118f8a5c3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.240661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da5cc60d-def0-4b8d-9aa6-ba5eb134a600]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.263468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96f14e2c-f09c-4aee-97b1-49a42a828472]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.286114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb75ef67-0d49-44ee-b2e0-397b0b13f134]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.288287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:27.289485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:27.290101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:21:27.290101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(90c12a68-55c9-4da5-b906-9be90ef49231) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:21:27.312231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:21:27.312658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c8439d4-ee82-4ee3-9112-6407a0bb29e4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.315648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c322023-a635-4de8-876a-57b70bc2a0be]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.316718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-90c12a68-55, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:27.354479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-90c12a68-55, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:27.355352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:27.355985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:27.356475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:27.357110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-90c12a68-55, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '90c12a68-55c9-4da5-b906-9be90ef49231', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:27:e8:c2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:27.424246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce58805d-bdec-441a-8a4f-9be9b5729891]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.487263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4b61dcbb-5231-4a51-a5c0-3009cb0b305e]: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:21:27.487263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:21:27.487263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:21:27.487263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:21:27.487263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:21:27.487263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:21:27.487263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:21:27.487263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:21:27.487263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:21:27.487263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:21:27.487263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:21:27.487263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:21:27.487263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:27.489243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b61dcbb-5231-4a51-a5c0-3009cb0b305e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:27.490267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:27.492191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:27.495395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9cdbfe9-fa56-4250-9bd5-7a1237c73ba6]: (4, ['qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.496264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:27.496649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:27.496952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:21:27.497359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:27.501157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87f1e8f0-68ca-4fad-8892-0024db4f0b75]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.506238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab77902a-902c-4318-8206-e2636218dda1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.510775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43475a97-4501-457b-aca7-dee0fdc3c1d6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.548648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[195ea80b-5c1a-4bf3-abd6-f6103d13953f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 578345, 'reachable_time': 17116, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188781, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.550475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:21:27.550760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9bf431a-1f22-4b2b-a347-015ada367516]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:27.551834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersNegativeTestJSON-718025888-router", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63" Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:47:d3:7f", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4700, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "90c12a68-55c9-4da5-b906-9be90ef49231", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b7c4afb-9649-49f5-b98f-7e26bf2b255d", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:27:e8:c2", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4704, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:06Z", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:24Z", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:21:27.554250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id 5cbb4b48-75e2-4895-8ac8-e329641ad12c. Wait time elapsed: 19.780 Jul 01 10:21:27.566062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id 5cbb4b48-75e2-4895-8ac8-e329641ad12c. Wait time elapsed: 19.780 Jul 01 10:21:27.566062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:28.660380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.107s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:28.661326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a2cfd54-251d-46f3-934d-906d043b085c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:21:28.662425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a2cfd54-251d-46f3-934d-906d043b085c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:21:28.662746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:28.663173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:28.664785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[daf776eb-341c-4941-8a35-eff98cf88329]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:28.679075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27667a13-cfe2-42ea-96eb-22eac989745a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:28.700719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bdc4f612-86e9-4aae-b879-402dfb139d76]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:28.732248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b07dab68-e2a4-492c-b221-97cdb9e10b20]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188821, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:28.755148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f2ffae0-2b5b-486f-87fb-628d236f6f88]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:28.777481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d8c6c86-d0b9-4af9-8c89-abec8c5c2d59]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:28.797658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24859412-22c1-43e2-83a2-e6d2999e640d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:28.819431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b44a25e-4253-453b-91c0-c4a5cf1ca91c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:28.819938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:28.820210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:28.820471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:21:28.820740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(90c12a68-55c9-4da5-b906-9be90ef49231) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:21:28.837327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:21:28.837755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ead750a-99a5-497d-82ef-6d6f94868038]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:28.841270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d567be67-2524-4619-ab43-26217f97aa43]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:28.842318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-90c12a68-55, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:28.881779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-90c12a68-55, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:28.883227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:28.883906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:28.884495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:28.885078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-90c12a68-55, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '90c12a68-55c9-4da5-b906-9be90ef49231', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:27:e8:c2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:28.938824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bd2c481-cd40-4f63-b697-7f4246f60e33]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:28.987885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cd40befd-b4da-4073-b5e3-4f61e8e6532a]: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:21:28.987885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:21:28.987885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:21:28.987885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:21:28.987885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:21:28.987885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:21:28.987885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:21:28.987885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:21:28.987885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:21:28.987885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:21:28.987885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:21:28.987885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:21:28.987885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:28.988831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd40befd-b4da-4073-b5e3-4f61e8e6532a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:28.989855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:28.993094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:28.997809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbc75e45-ae96-4683-8b8d-9ecf5c2d6c7b]: (4, ['qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:28.998976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:28.999583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:29.000056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:21:29.000528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:29.001889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b239321a-9d42-4284-9661-61c8cea8bdea]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:29.003427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f79873d-34af-4494-9947-b03dc7396761]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:29.006126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[864d9ac5-92e8-4b2d-98c2-e143f60e0988]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:29.043260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e52277e7-c2d6-42a1-ab2c-51f9bc0542a7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 578499, 'reachable_time': 22964, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188839, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:29.059077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:21:29.059528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c4dd187-808d-4fae-ab7c-ca977c8aff7f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:29.060736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersNegativeTestJSON-718025888-router", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63" Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:47:d3:7f", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4700, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "90c12a68-55c9-4da5-b906-9be90ef49231", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b7c4afb-9649-49f5-b98f-7e26bf2b255d", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:27:e8:c2", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4704, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:06Z", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:27Z", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:21:29.063691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id b7cb9f64-5bb3-4822-b756-8117d970b73c. Wait time elapsed: 23.292 Jul 01 10:21:29.073783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id b7cb9f64-5bb3-4822-b756-8117d970b73c. Wait time elapsed: 23.292 Jul 01 10:21:29.073783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:31.619679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.557s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:31.620240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a2cfd54-251d-46f3-934d-906d043b085c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:21:31.621409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a2cfd54-251d-46f3-934d-906d043b085c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:21:31.621744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:31.622029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:31.623216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd971e34-d213-4846-89d4-6cb87e7fd0a9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.637742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58952d2e-a2db-4b98-921f-96b72be1b497]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.661379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9447986d-f6c1-42be-a051-183966fbe186]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.699335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ace03674-2db4-4df0-8ceb-e403bb7995dc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188845, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.722271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f3e65dd-40f5-4a6e-80ed-21d4ddeaa654]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.744172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d5d8027-ab6b-4c65-9b68-e11ea9719ea6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.766083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3999c732-1d1b-4fba-8417-b36647860837]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.787102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b3edf15-fea2-426d-9efd-044343f4388b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.787789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:31.788132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:31.788601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:21:31.788884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(90c12a68-55c9-4da5-b906-9be90ef49231) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:21:31.804765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:21:31.805154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f31b1ba3-9873-47e0-82aa-37926da3ca13]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.808386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b600bcb-6df8-405a-89f9-4773bb572e97]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.809357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-90c12a68-55, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:31.850370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-90c12a68-55, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:31.851083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:31.851556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:31.851964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:31.852335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-90c12a68-55, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '90c12a68-55c9-4da5-b906-9be90ef49231', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:27:e8:c2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:31.911977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4be2be71-55dc-48c9-af24-dab3af415bcc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.948824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[511b4ab9-e7b1-435e-b4d7-0f156e1458f0]: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:21:31.948824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:21:31.948824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:21:31.948824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:21:31.948824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:21:31.948824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:21:31.948824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:21:31.948824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:21:31.948824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:21:31.948824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:21:31.948824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:21:31.948824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:21:31.948824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:31.949925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[511b4ab9-e7b1-435e-b4d7-0f156e1458f0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:31.950158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:31.952998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:31.956588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8d2f373-7e19-4a17-bef3-1786876feb6e]: (4, ['qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.957400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:31.957726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:31.957982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:21:31.958325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:31.959442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34d8d1f3-f3c1-42f5-8e24-c67485595d93]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.960650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ba1aa2a-52d4-4e9c-988c-a19fe8249a4c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.962053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6bc5ef2-f20c-45d5-8321-8121881c2e4f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.985658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[449e8f84-1b00-4d03-8e77-1a025f02fcd7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 578796, 'reachable_time': 36640, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188863, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.988286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:21:31.988678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8765089-0110-45c8-b83b-918a39248908]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:31.989790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersNegativeTestJSON-718025888-router", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63" Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:47:d3:7f", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4700, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "90c12a68-55c9-4da5-b906-9be90ef49231", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b7c4afb-9649-49f5-b98f-7e26bf2b255d", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:27:e8:c2", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4704, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:06Z", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:29Z", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:21:31.994008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:21:32.008955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3 Jul 01 10:21:32.008955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id 5cbb4b48-75e2-4895-8ac8-e329641ad12c. Wait time elapsed: 24.222 Jul 01 10:21:32.008955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 2, update_id 5cbb4b48-75e2-4895-8ac8-e329641ad12c. Wait time elapsed: 24.222 Jul 01 10:21:32.008955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:32.739183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.743s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:32.742021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0a2cfd54-251d-46f3-934d-906d043b085c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:21:32.743163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0a2cfd54-251d-46f3-934d-906d043b085c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:21:32.743772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:32.744893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:32.747486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9917fee5-f297-4bcb-bc4c-0e839bdd2cd2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:32.781632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6da02a00-e4a2-4241-b593-5d34db41587e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:32.867009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa40a674-dac2-423e-b339-0aee007c89ea]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:32.927840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb7a709e-1aa9-40f5-a83b-f5459f85da42]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 188907, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:32.976943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e66d4b0a-a2f7-4fad-85b6-bea8a12468cb]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.016266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25b671bc-b8e0-425e-bf15-944400a74d3d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.045616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52b8413a-8f91-4965-af19-87cb0a7d939b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.070396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[791f2f5e-4eaf-46ca-a9e5-4fd7623aa21d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.071227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:33.071704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:33.072171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:21:33.072680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(90c12a68-55c9-4da5-b906-9be90ef49231) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:21:33.095791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:21:33.096047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[394ca1d7-73ba-46e6-91cf-5bae1410dc77]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.099237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6da5bb6d-fb71-4ae1-bec0-f44ddb915131]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.100361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-90c12a68-55, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:33.126927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-90c12a68-55, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:33.127662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:33.128225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:33.128756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-90c12a68-55, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:33.129603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-90c12a68-55, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '90c12a68-55c9-4da5-b906-9be90ef49231', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:27:e8:c2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:21:33.184835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3769a48-e7b6-4087-bf65-e25d2048c513]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.220188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a660d953-582c-46d7-bfd4-41fb9859d8e5]: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:21:33.220188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:21:33.220188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:21:33.220188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:21:33.220188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:21:33.220188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:21:33.220188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:21:33.220188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:21:33.220188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:21:33.220188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:21:33.220188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:21:33.220188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:21:33.220188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:33.221010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a660d953-582c-46d7-bfd4-41fb9859d8e5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:33.221310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:33.224198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:33.226038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2529f390-215a-468b-b7ae-86b96e06cc03]: (4, ['qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.226401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:33.226786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:33.227206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:21:33.227595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0a2cfd54-251d-46f3-934d-906d043b085c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:21:33.228700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec2b3eb7-c321-4d2e-b3ee-0b2846f932a4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.230382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6727125-ce71-4d59-9ef9-ba046a7e5ab6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.231475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ce6a6a6-477f-4595-ae7c-ad5377689ec6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.255739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ddb9f2d-e232-4ae4-a2ba-761613e736fe]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 578918, 'reachable_time': 22172, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 188931, 'error': None, 'target': 'qrouter-0a2cfd54-251d-46f3-934d-906d043b085c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.259793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:21:33.262835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78320376-523c-4120-9797-dce8e83e9da8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0a2cfd54-251d-46f3-934d-906d043b085c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-90c12a68-55 not found in namespace qrouter-0a2cfd54-251d-46f3-934d-906d043b085c. Jul 01 10:21:33.264164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-ServersNegativeTestJSON-718025888-router", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63" Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c48059e4-5f56-46bb-b0e1-16c5fda51a96", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:47:d3:7f", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4700, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.63", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:04Z", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:07Z", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "90c12a68-55c9-4da5-b906-9be90ef49231", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2b7c4afb-9649-49f5-b98f-7e26bf2b255d", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:27:e8:c2", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0a2cfd54-251d-46f3-934d-906d043b085c", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4704, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:21:06Z", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:21:31Z", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 21, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "64c2cc6da2b34d2097952e6d7e7c6a3c", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "79c6a82a-455b-4c6f-9591-61441b803c21", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:21:33.267048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3 Jul 01 10:21:35.258160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:21:52.934490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-e6d1147e-5f1d-49ef-a575-6a553b6a23d1 admin admin] Got routers updated notification :['0a2cfd54-251d-46f3-934d-906d043b085c'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:21:52.936424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 1, update_id aebe7016-575c-4594-97c6-54a4e2d36fc9. Wait time elapsed: 0.001 Jul 01 10:21:52.936680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 3, priority 1, update_id aebe7016-575c-4594-97c6-54a4e2d36fc9. Wait time elapsed: 0.001 Jul 01 10:21:52.936977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:21:53.922548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c49b49f2-3560-418c-aeda-7a6d88331bdc admin admin] Got router deleted notification for 0a2cfd54-251d-46f3-934d-906d043b085c {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:21:54.142893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.206s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:21:54.143528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 0a2cfd54-251d-46f3-934d-906d043b085c was not found. Performing router cleanup Jul 01 10:21:54.145141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9253f72-d162-475d-8df9-51eaef0b5abf]: (4, ['qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:54.145707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:54.146131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:54.146512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 0a2cfd54-251d-46f3-934d-906d043b085c, update_id aebe7016-575c-4594-97c6-54a4e2d36fc9. Time elapsed: 1.210 Jul 01 10:21:54.146900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0a2cfd54-251d-46f3-934d-906d043b085c, action 1, priority 1, update_id 742a54e6-8bb9-4038-8104-dcad561465a7. Wait time elapsed: 0.224 Jul 01 10:21:54.147135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0a2cfd54-251d-46f3-934d-906d043b085c, action 1, priority 1, update_id 742a54e6-8bb9-4038-8104-dcad561465a7. Wait time elapsed: 0.224 Jul 01 10:21:54.147538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 0a2cfd54-251d-46f3-934d-906d043b085c was not found. Performing router cleanup Jul 01 10:21:54.148705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ebe41adc-20b7-484c-88dc-78472e81e542]: (4, ['qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:21:54.149490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:21:54.149756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:21:54.149996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 0a2cfd54-251d-46f3-934d-906d043b085c, update_id 742a54e6-8bb9-4038-8104-dcad561465a7. Time elapsed: 0.003 Jul 01 10:22:11.591399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c53a5de5-1e11-4805-b69d-59d4fdbb8ebb tempest-RoutersNegativeTest-776485270 tempest-RoutersNegativeTest-776485270-project] Got router deleted notification for c6e94399-e310-44ac-a593-a0636e32f82c {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:22:11.593757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c6e94399-e310-44ac-a593-a0636e32f82c, action 1, priority 1, update_id 982480e0-89ba-4632-bb56-e31eef036049. Wait time elapsed: 0.001 Jul 01 10:22:11.594591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c6e94399-e310-44ac-a593-a0636e32f82c, action 1, priority 1, update_id 982480e0-89ba-4632-bb56-e31eef036049. Wait time elapsed: 0.001 Jul 01 10:22:11.595030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router c6e94399-e310-44ac-a593-a0636e32f82c was not found. Performing router cleanup Jul 01 10:22:11.596925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef0379bd-494f-41d1-b74b-893ce2d8fdb7]: (4, ['qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:11.597871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:11.598216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:11.598538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for c6e94399-e310-44ac-a593-a0636e32f82c, update_id 982480e0-89ba-4632-bb56-e31eef036049. Time elapsed: 0.005 Jul 01 10:22:15.139211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:15.140090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:15.140589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:15.140992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:15.211832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-2cb63268-2af5-4aa6-b672-bcf2fa32c91a admin admin] Got routers updated notification :['06ce6427-dab9-46fd-8b57-1c3f490f0250'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:22:15.213603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 1, update_id e5f26787-8340-4087-8d78-e2d95be65fe4. Wait time elapsed: 0.000 Jul 01 10:22:15.213603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 1, update_id e5f26787-8340-4087-8d78-e2d95be65fe4. Wait time elapsed: 0.001 Jul 01 10:22:15.213603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:15.260072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:22:16.488260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.275s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:16.488891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 06ce6427-dab9-46fd-8b57-1c3f490f0250 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:16.489975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:16.490531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:16.491186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:16.493238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72675473-ea6b-4a9a-8c4b-4217aca48396]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:16.513820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78a3e61a-c9ef-4915-8125-ad28107e9728]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:16.539190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92b664cd-2558-49fe-b6f0-11712c2f6df3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:16.584609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbc63e10-89b9-4ac3-acc4-62b71aac3edd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 189501, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:16.606817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9ca4be4-6b04-48dd-bcf5-4a60c0758d49]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:16.628270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb8c4a74-b265-4bae-b49d-366cdd5e51d8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:16.655278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0db362cc-e8d9-4726-b73a-338b5a092ec7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:16.679375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1776cf0c-1996-47e5-8b24-c0491de5df7e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:16.681020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:16.682014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:16.682661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:16.704610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f47b4c53-9c92-49d8-9efb-641d967b78e5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 583284, 'reachable_time': 23432, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189507, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:16.705566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:16.705927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:16.706570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:22:16.706898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:22:16.707505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:16.707995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:16.708398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '6e424432-aa98-4b1f-bfdc-786980c6bd5f', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:10:e3:5c', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '06ce6427-dab9-46fd-8b57-1c3f490f0250', 'device_owner': 'network:router_gateway', 'standard_attr_id': 4800, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.9', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:22:14Z', 'updated_at': '2022-07-01T10:22:16Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-6e424432-aa), ns(qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:22:16.726377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-6e424432-aa not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:16.726753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63dba957-59b4-45eb-a9e7-d224d0dea874]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:16.730683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6def4b34-f941-4efd-9294-68ca2ecab4ea]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:16.731837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-6e424432-aa, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:16.732232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:22:16.733208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-6e424432-aa, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:16.734117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-6e424432-aa, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:16.734491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-6e424432-aa, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:16.735054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-6e424432-aa, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:16.735442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-6e424432-aa, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '6e424432-aa98-4b1f-bfdc-786980c6bd5f', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:10:e3:5c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:16.780953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[513c4fda-47e6-4f0f-88ee-1281b3d58dfa]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:16.820856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[19f7a380-2a56-4857-8f41-c435dc48de2d]: Network interface qg-6e424432-aa not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:16.820856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:16.820856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:16.820856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:16.820856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:16.820856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:16.820856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:16.820856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:16.820856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:16.820856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:16.820856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:16.820856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:16.820856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6e424432-aa not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:16.822775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19f7a380-2a56-4857-8f41-c435dc48de2d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-6e424432-aa not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:16.823540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:22:16.824213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:22:16.848295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0dc59638-c31a-40ad-88aa-2262d8ca4db7]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:17.022782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b30d7c36-a929-4a4f-8cb0-601958644c17]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:17.049164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ff3468d-0411-4d4d-9f06-cba1bc28957d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:17.240555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00472c55-15a8-4a25-a52c-6c94824cdd06]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:17.241278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:22:17.241748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:22:17.242196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:22:17.242636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" released by "process_external" :: held 0.536s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-6e424432-aa not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6e424432-aa not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6e424432-aa not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:17.243727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6e424432-aa not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6e424432-aa not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:17.248068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:17.250975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1aad1527-76f9-43fb-8ac1-aa923921e2bf]: (4, ['qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:17.250975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:17.250975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:17.251260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:17.251522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:17.252763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48d96f6c-7040-46e9-9fac-e81af0b5c6af]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:17.253692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1807b712-a527-4fbf-8a42-52ef24051b7a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:17.255353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2d55fab-28aa-4a6f-ab34-502d64ba7650]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:17.273672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2e470af-6f87-444b-8e31-85b501ab92af]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 583284, 'reachable_time': 23432, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189545, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:17.279832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:17.280582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[352337db-7587-4a59-8d43-e6472901a252]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6e424432-aa not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-6e424432-aa not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:17.281776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-583633917-router", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:13Z", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:14Z", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:10:e3:5c", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4800, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:14Z", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:16Z", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id e5f26787-8340-4087-8d78-e2d95be65fe4. Wait time elapsed: 2.071 Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id e5f26787-8340-4087-8d78-e2d95be65fe4. Wait time elapsed: 2.071 Jul 01 10:22:17.285273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:17.843671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ec0d0280-eecd-446a-ac33-423dd7b0d572 admin admin] Got routers updated notification :['06ce6427-dab9-46fd-8b57-1c3f490f0250'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:22:20.795145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-9b4c6927-eaa3-46f5-bb3d-58b4e07012ba admin admin] Got routers updated notification :['8f483531-0156-4109-94b6-07df8031167f'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:22:20.796760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 1, update_id 7ee424a3-69bd-4bf2-9a28-a0bbedf6e145. Wait time elapsed: 0.001 Jul 01 10:22:20.797128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 1, update_id 7ee424a3-69bd-4bf2-9a28-a0bbedf6e145. Wait time elapsed: 0.001 Jul 01 10:22:20.797640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:21.933167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.650s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:21.934316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 06ce6427-dab9-46fd-8b57-1c3f490f0250 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:21.935684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:21.936874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:21.937429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:21.939773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00a2149a-504d-4d47-98fb-d949a119413d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:21.954846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6571df10-d6f9-436b-a084-9be60e0bb36f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:21.977463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf7283cc-f0ce-4da5-80ac-4d35ca22aaa9]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.017800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6134a0f5-4782-438f-8cec-6035aea688a4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 189616, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.034307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.237s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:22.034573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 8f483531-0156-4109-94b6-07df8031167f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:22.036526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (8f483531-0156-4109-94b6-07df8031167f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:22.036856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:22.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:22.038569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e905f6c-6a21-4fec-85df-61f3c52371de]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.042663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abb328b2-9652-41a2-a422-ae18947f933c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.072792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d401df2-3f1a-4b74-b135-201d6a7a1dc5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.082845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64258e99-aac9-45ad-af2e-4eec8faefe1f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.110491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a75c3b8a-bcc4-4132-a868-bd5b507a66c1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.115971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe7a36a4-4e44-4c4b-b290-6a7a641c3c98]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.140705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08cd065e-2b1c-4c36-ad12-685a059ab70f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.141511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:22.142506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:22.142797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:22.143698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(46c82d52-0298-422e-871d-ddf9005b2611) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:22.144400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29d0332a-6bbd-42fe-ac8f-f5ac7f15ca71]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 189623, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.169071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:22.169256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84e3de00-d17a-400a-b76b-0673c0605632]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.173569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cf1cf57-2994-4074-945c-f7be2cbea36a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.174612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-46c82d52-02, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:22.175053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:22:22.175937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-46c82d52-02, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:22.177221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:22.177897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:22.178364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:22.178892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-46c82d52-02, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '46c82d52-0298-422e-871d-ddf9005b2611', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:22:41:9e'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:22.180618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e77524a-57f5-412e-8020-87f5416eb9e6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.220532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d016148-89cc-4893-ad83-196b22493ab6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.243441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46a41b3e-60b3-4a74-b2c0-11e985986264]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.253256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[45241c5d-eaa1-46ff-a54c-e8435ef6f21d]: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:22.253256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:22.253256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:22.253256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:22.253256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:22.253256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:22.253256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:22.253256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:22.253256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:22.253256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:22.253256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:22.253256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:22.253256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:22.254714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45241c5d-eaa1-46ff-a54c-e8435ef6f21d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:22.255404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:22.257006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:22.259165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[434a1823-1ada-45e5-8a41-cae86315a775]: (4, ['qrouter-8f483531-0156-4109-94b6-07df8031167f', 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.260692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:22.261078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:22.261343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:22.261752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:22.263783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[661d1b52-cb16-4cd8-a1a8-e1314789958d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.265428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58025bcf-8f87-4e27-b22c-c00aa0e72126]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.266179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c9af662-86e3-4f41-9faf-e7c5ecc01c10]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.274584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62df2599-070d-428c-9249-3e17a2a9feb1]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.293400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[170f0005-3a66-4cf2-87f4-83210c1e6f73]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 583828, 'reachable_time': 39664, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189643, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.300365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:22.300673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[721f4822-1fc1-46ed-bda4-88eaf429ac5d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:22.301684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-583633917-router", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:13Z", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:16Z", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:10:e3:5c", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4800, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:14Z", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:19Z", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "46c82d52-0298-422e-871d-ddf9005b2611", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "4e509fcf-4f66-429c-9cb9-2d53c43b2b37", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:22:41:9e", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4804, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:15Z", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:19Z", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:22.304535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 1, update_id db3e68ce-65c0-4eab-89ff-e34465f4eef7. Wait time elapsed: 4.459 Jul 01 10:22:22.318292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 1, update_id db3e68ce-65c0-4eab-89ff-e34465f4eef7. Wait time elapsed: 4.459 Jul 01 10:22:22.318292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:22.318292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a63503e8-1fa3-4a58-8af9-b3bdb44c6738]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.318292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:22.318292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:22.318292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:22.342836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29032a06-6571-46b1-bc5d-6fb157dff2bc]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 583840, 'reachable_time': 22264, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189645, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.343650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:22.343930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:22.344387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-8f483531-0156-4109-94b6-07df8031167f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:22:22.344651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-8f483531-0156-4109-94b6-07df8031167f" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:22:22.344984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:22.345245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:22.345561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '29417014-056d-4f90-980f-1945d1a5e51e', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:cf:9c:51', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '8f483531-0156-4109-94b6-07df8031167f', 'device_owner': 'network:router_gateway', 'standard_attr_id': 4814, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.242', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:22:19Z', 'updated_at': '2022-07-01T10:22:21Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-29417014-05), ns(qrouter-8f483531-0156-4109-94b6-07df8031167f) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:22:22.365113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-29417014-05 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:22.365399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfc4e8eb-5575-4656-bc42-ad9478d45024]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.368907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[978285d1-e1d0-4d94-b4ca-b66df6a9bd60]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.370069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-29417014-05, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:22.370554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:22:22.371495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-29417014-05, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:22.372299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-29417014-05, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:22.372879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-29417014-05, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:22.373420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-29417014-05, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:22.373938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-29417014-05, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '29417014-056d-4f90-980f-1945d1a5e51e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:9c:51'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:22.434111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e22f3dd3-36f7-4e71-817b-d4e8fbed6053]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.468900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[30478d4d-edb7-4e45-b1c7-4bbef94b7851]: Network interface qg-29417014-05 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:22.468900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:22.468900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:22.468900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:22.468900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:22.468900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:22.468900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:22.468900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:22.468900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:22.468900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:22.468900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:22.468900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:22.468900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-29417014-05 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:22.469825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30478d4d-edb7-4e45-b1c7-4bbef94b7851]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-29417014-05 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.469825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-8f483531-0156-4109-94b6-07df8031167f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:22:22.470303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-8f483531-0156-4109-94b6-07df8031167f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:22:22.496736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[736dc5a3-674e-4716-a20d-c6de0f64f35f]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.690461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df383c6b-c9ae-4792-8021-d8acf8dc6787]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.703460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e89cddc-cd7c-4328-a7da-11e9ea0e56bc]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.874871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81a60d35-8df3-49a1-bf32-4694998c0d85]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.875824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:22:22.876222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-8f483531-0156-4109-94b6-07df8031167f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:22:22.876598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-8f483531-0156-4109-94b6-07df8031167f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:22:22.877083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-8f483531-0156-4109-94b6-07df8031167f" released by "process_external" :: held 0.532s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-29417014-05 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-29417014-05 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-29417014-05 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:22.878298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-29417014-05 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-29417014-05 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:22.882288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:22.895979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb30ed8a-1d78-4e76-ab31-524598274e74]: (4, ['qrouter-8f483531-0156-4109-94b6-07df8031167f', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.897179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:22.897601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:22.897919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:22.898349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:22.900836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7538411d-c712-43a5-ba9e-7aa03bd2bae6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.902264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19f1aaa8-0801-417d-88df-d08c8ca4ec79]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.903534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5a65af7-cb3a-4323-9a99-80729e5bc507]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.935976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79cfad4b-7cb1-46c3-9460-c8a8ad515c11]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 583840, 'reachable_time': 22264, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189665, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.943117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-8f483531-0156-4109-94b6-07df8031167f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:22.943396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b04ccd94-5e6d-49ee-bc01-077fa8d2d4f0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-29417014-05 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-29417014-05 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:22.944535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-1253834789-router", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242" Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:20Z", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:9c:51", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4814, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:21Z", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 7ee424a3-69bd-4bf2-9a28-a0bbedf6e145. Wait time elapsed: 2.149 Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 7ee424a3-69bd-4bf2-9a28-a0bbedf6e145. Wait time elapsed: 2.150 Jul 01 10:22:22.950114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:23.254539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-1623c6bd-0a1d-4818-a08b-4219e6508953 admin admin] Got routers updated notification :['8f483531-0156-4109-94b6-07df8031167f'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:22:24.664939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.719s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:24.665526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 8f483531-0156-4109-94b6-07df8031167f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:24.666147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (8f483531-0156-4109-94b6-07df8031167f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:24.666504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:24.666826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:24.668182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13e99fab-1ca9-41c3-bf5d-e072c9e85b4c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:24.687928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2f8f43e-f535-4405-b033-57472aadc103]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:24.711197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6eece7b5-caf6-40c9-8ef6-0527754424e5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:24.776463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a34fb322-d52b-42b5-a708-fe0519a8bc60]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 189731, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:24.802378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54fc6124-fb67-43b9-9ff0-3248ecc3a192]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:24.826998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5c9c7dd-f3bc-4f12-8bb1-f8a3ca94b0f1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:24.852039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a38d2c2-9e14-4c04-8f36-47de7e3ac017]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:24.877362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae64aa00-db8d-41f5-aebb-ae42ffaf8ffe]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:24.878079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:24.878508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:24.879078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:24.879632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e47dc9fc-8887-4123-86b9-b28f2b8be563) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:24.898450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:24.898832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[992f2934-4f4c-4287-ad3b-6a74a1056f66]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:24.904430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ab6c262-cd1e-4323-8efd-f26dbe9dbf80]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:24.905621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e47dc9fc-88, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:24.906176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:22:24.907086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e47dc9fc-88, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:24.908134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:24.908518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:24.909039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:24.909557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e47dc9fc-88, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e47dc9fc-8887-4123-86b9-b28f2b8be563', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:35:73'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:24.967879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14963bf8-11bb-4b0e-b587-ce953d14899c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.007574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c1cb3a07-db87-4e54-a046-a08ae25e0aff]: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:25.007574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:25.007574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:25.007574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:25.007574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:25.007574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:25.007574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:25.007574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:25.007574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:25.007574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:25.007574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:25.007574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:25.007574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:25.008542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1cb3a07-db87-4e54-a046-a08ae25e0aff]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:25.009380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:25.012273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:25.014824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef1f6ec5-9b37-4fc1-8e44-fd155df55b8e]: (4, ['qrouter-8f483531-0156-4109-94b6-07df8031167f', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.016078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:25.016467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:25.016725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:25.017135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:25.018188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c4dc785-e5da-4203-baac-ac63137556c0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.019517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c845d1e7-dfe8-49da-aa19-38a17aa490f9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.020450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4f1eb6c-a50f-4291-81ef-eb10d6b03db5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.046560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7297ed8c-8feb-448e-9ec6-3450972bda7b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 584103, 'reachable_time': 44764, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189748, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.049793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-8f483531-0156-4109-94b6-07df8031167f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:25.050138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c790677-6db6-4110-8257-2771181335d7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:25.051266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-1253834789-router", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242" Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:22Z", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:9c:51", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4814, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:21Z", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e47dc9fc-8887-4123-86b9-b28f2b8be563", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "eea6a246-41a4-4c29-9aa5-cba81e33c4e5", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:35:73", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4819, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:21Z", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:24Z", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:25.054261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:25.068206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 1, update_id 80149f1b-838b-40a6-a304-75bb5e595cd7. Wait time elapsed: 1.804 Jul 01 10:22:25.068206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 1, update_id 80149f1b-838b-40a6-a304-75bb5e595cd7. Wait time elapsed: 1.805 Jul 01 10:22:25.068206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:25.558870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.255s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:25.559036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 06ce6427-dab9-46fd-8b57-1c3f490f0250 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:25.560244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:25.560511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:25.560917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:25.562807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7aea373-dda1-44db-9437-51cd05293e3f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.581807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[513da52f-849c-4456-aa3e-160427dbbd9f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.618414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4aef5b02-6fb1-460e-8a2e-f81dd06bd34b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.649541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b9ab65f-e503-485d-9bf9-205742692b0d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 189751, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.672274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7141328b-6adc-4810-af06-7f5442dd691d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.695277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30529aee-8ce9-414d-926f-f3446e44466b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.720223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[672bce49-4a75-4efa-a38b-52fde1ce3134]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.741468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46ad3abf-cf31-4fdd-9780-c185ef0e6761]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.742023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:25.742293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:25.742555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:25.742832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(46c82d52-0298-422e-871d-ddf9005b2611) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:25.766440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:25.766739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddb9064b-750c-4a16-9ce1-ab4803f2953e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.770110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28ed668d-7c6f-4698-a4ba-9791178312f7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.771287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-46c82d52-02, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:25.826689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-46c82d52-02, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:25.827727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:25.831066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:25.831708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:25.832322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-46c82d52-02, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '46c82d52-0298-422e-871d-ddf9005b2611', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:22:41:9e'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:25.916270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[894f6539-afe6-4538-af72-619894f6c5f3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[84f95c62-3740-4ddb-b17e-cd3dc67aea0e]: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:25.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:25.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:25.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:25.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:25.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:25.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:25.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:25.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:25.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:25.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:25.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:25.976830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:25.979287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84f95c62-3740-4ddb-b17e-cd3dc67aea0e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:25.980502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:25.983009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:25.985218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b27fa8c8-e154-493e-88b6-34a95eaf884e]: (4, ['qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.985673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:25.985987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:25.986413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:25.986969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:25.988475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a47f5d49-b060-48df-9e35-b3aa9317888d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.989652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d235226b-52bf-4a99-ac4b-b74c0cae16be]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:25.990810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a17b27c-310d-40b9-8f6b-f0966ec3bcd3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:26.020571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d13a2bf5-c1c8-46ab-a386-6b04ff070793]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 584191, 'reachable_time': 18072, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189769, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:26.032659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:26.032886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de305b36-a582-424e-a9a1-3de7887283ed]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:26.033925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-583633917-router", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:13Z", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:16Z", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:10:e3:5c", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4800, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:14Z", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:19Z", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "46c82d52-0298-422e-871d-ddf9005b2611", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "4e509fcf-4f66-429c-9cb9-2d53c43b2b37", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:22:41:9e", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4804, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:15Z", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:19Z", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:26.036401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id e5f26787-8340-4087-8d78-e2d95be65fe4. Wait time elapsed: 10.823 Jul 01 10:22:26.050949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id e5f26787-8340-4087-8d78-e2d95be65fe4. Wait time elapsed: 10.823 Jul 01 10:22:26.050949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:26.902038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.866s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:26.903014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 06ce6427-dab9-46fd-8b57-1c3f490f0250 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:26.905775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:26.906488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:26.906843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:26.908192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a5daecc-6b3a-4a61-ae57-14e52b81ad5c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:26.940280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e123fbd-d75f-4c90-8728-69c8c3c45869]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:26.967197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ee9ad5c-6fac-4c06-9104-b83799b8ba4f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.004005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d54fded6-db3d-4146-bb45-d45fdfef97c9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 189774, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.025688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13c4f956-7e28-4223-ae47-54ea4db3d6bd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.048392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77d1866a-c18c-4058-b75c-f9d8704c855f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.074059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8170d13c-d95b-486e-9675-cdfa0e0d69d0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.097627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2c80e68-5a43-4b85-beb1-b591c6709357]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.098275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:27.098619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:27.098953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:27.099300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(46c82d52-0298-422e-871d-ddf9005b2611) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:27.121749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:27.122416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a58a746-6429-469c-9420-db6ca0aa330e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.125660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a2ca0fa-a0f4-4874-ba9d-d8650bba6af0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.126832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-46c82d52-02, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:27.174024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-46c82d52-02, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:27.174983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:27.176419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:27.176419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:27.176419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-46c82d52-02, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '46c82d52-0298-422e-871d-ddf9005b2611', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:22:41:9e'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:27.235007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f01e423-d8e8-49bf-b945-997a706d6760]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.271691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1d6ef8b7-9e57-4bd7-bece-73564c21754c]: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:27.271691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:27.271691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:27.271691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:27.271691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:27.271691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:27.271691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:27.271691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:27.271691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:27.271691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:27.271691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:27.271691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:27.271691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:27.273179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d6ef8b7-9e57-4bd7-bece-73564c21754c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:27.274232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:27.277116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:27.278928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a73f249d-47ad-49b5-9e4d-1dd61cc35671]: (4, ['qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.279176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:27.279517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:27.279767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:27.280100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:27.281138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26bff9fa-1e02-4554-bc88-dacd5ade3fbc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.282501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[958e2508-ca85-4373-ba4b-967678ee223c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.283481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed845a6d-4f6f-4748-a2f5-58432ae3677e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.305510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26a3da5a-f22b-4ad3-a7c9-5b2c8c62a238]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 584326, 'reachable_time': 32164, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189793, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.315312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:27.315708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f88a361-ed65-48b1-858c-021a905d239e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:27.316850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-583633917-router", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:13Z", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:16Z", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:10:e3:5c", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4800, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:14Z", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:19Z", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "46c82d52-0298-422e-871d-ddf9005b2611", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "4e509fcf-4f66-429c-9cb9-2d53c43b2b37", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:22:41:9e", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4804, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:15Z", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:25Z", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:27.319333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id db3e68ce-65c0-4eab-89ff-e34465f4eef7. Wait time elapsed: 9.474 Jul 01 10:22:27.330843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id db3e68ce-65c0-4eab-89ff-e34465f4eef7. Wait time elapsed: 9.474 Jul 01 10:22:27.330843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:28.207162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.147s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:28.207815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 8f483531-0156-4109-94b6-07df8031167f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:28.208470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (8f483531-0156-4109-94b6-07df8031167f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:28.208802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:28.209088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:28.210274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e339bef-909a-43ef-a875-2d5192d654c7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.224321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[573a2830-b203-4c5a-bf5b-a1ead590e57c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.246775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35db5161-2e01-43ff-9eb1-b272cbe7c629]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.277598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b816657-912c-415e-9efd-4476a948f5a2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 189796, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.289993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d66c1ad-d544-43d0-863d-6d4143b57e35]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.316738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e821d6da-3933-4ea9-84dd-c13f4d2dd74f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.349928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a624610a-fcd7-4413-b4ef-c16f5ace8319]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.392786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[302a03b4-27d3-4eb5-9f43-b7366cdb5c7c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.394003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:28.394495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:28.395825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:28.396188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e47dc9fc-8887-4123-86b9-b28f2b8be563) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:28.422047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:28.422347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b4c7348-1e3a-4fc1-a94c-d7b44940cf62]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.425016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03c02f96-df28-471a-b992-4fc02c6f9b0a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.426186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e47dc9fc-88, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:28.460091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e47dc9fc-88, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:28.460991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:28.461576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:28.461945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:28.462330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e47dc9fc-88, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e47dc9fc-8887-4123-86b9-b28f2b8be563', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:35:73'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:28.524200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e9610b6-7348-4649-992d-270ac2c89083]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.563749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[63f36770-aea2-4f85-a857-002d26387f03]: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:28.563749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:28.563749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:28.563749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:28.563749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:28.563749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:28.563749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:28.563749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:28.563749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:28.563749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:28.563749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:28.563749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:28.563749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:28.564775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63f36770-aea2-4f85-a857-002d26387f03]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:28.565037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:28.566867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:28.569119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed50cb5b-6fd6-4d6f-b286-825be0b2f8c3]: (4, ['qrouter-8f483531-0156-4109-94b6-07df8031167f', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.569119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:28.569119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:28.569119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:28.569484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:28.570258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d45d9602-66ca-4540-85c6-6775f96b1afb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.571288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af3658cf-adfc-4e1c-a045-87ab29f2c4b9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.572919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d7cdf0b-76f6-48f0-b2e4-7e1078b76059]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.593368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[388533a4-ccdc-4180-a4d7-29100079a5e2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 584454, 'reachable_time': 34268, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189816, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.595289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-8f483531-0156-4109-94b6-07df8031167f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:28.595621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21ee8a26-4b9d-46e3-8769-d18b02db6fe3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:28.596669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-1253834789-router", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242" Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:22Z", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:9c:51", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4814, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:26Z", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e47dc9fc-8887-4123-86b9-b28f2b8be563", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "eea6a246-41a4-4c29-9aa5-cba81e33c4e5", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:35:73", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4819, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:21Z", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:24Z", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:28.599026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 7ee424a3-69bd-4bf2-9a28-a0bbedf6e145. Wait time elapsed: 7.802 Jul 01 10:22:28.612094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 7ee424a3-69bd-4bf2-9a28-a0bbedf6e145. Wait time elapsed: 7.802 Jul 01 10:22:28.612094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:29.278867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.680s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:29.279499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 8f483531-0156-4109-94b6-07df8031167f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:29.280607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (8f483531-0156-4109-94b6-07df8031167f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:29.280951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:29.281414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:29.282727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[215a8e30-e7c7-42b8-ad78-24d3152ed08a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.294777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[adc63e1a-fa18-43af-8c46-5204fea3fdf9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.315354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95d8d3fe-5261-497d-b9a5-73bda56eccff]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.346264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c980b4d0-a59e-470a-aa41-61674613a423]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 189819, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.367346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0a2e1ec-a6f2-46af-895a-f9caecc547b4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.387627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e892159-4f4b-4055-afc9-438c903ab974]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.408899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85ff1020-2484-44e7-9b63-9b29250145ee]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.430311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a79438ad-0d29-4a4e-b1a2-9caccc8edb3e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.431094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:29.432040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:29.432646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:29.432930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e47dc9fc-8887-4123-86b9-b28f2b8be563) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:29.450473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:29.450764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98707932-36d0-4cc9-8c5a-5628e1c5911b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.453379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fef261b0-2db1-46b7-abb0-dbdc23e72b03]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.454376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e47dc9fc-88, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:29.489796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e47dc9fc-88, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:29.490740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:29.491139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:29.491542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:29.492064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e47dc9fc-88, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e47dc9fc-8887-4123-86b9-b28f2b8be563', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:35:73'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:29.538073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37254ebb-b91a-43f5-895f-e05a0f4da014]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.588524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[783e5123-4676-4d71-883a-6e4ebd081485]: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:29.588524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:29.588524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:29.588524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:29.588524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:29.588524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:29.588524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:29.588524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:29.588524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:29.588524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:29.588524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:29.588524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:29.588524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:29.589397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[783e5123-4676-4d71-883a-6e4ebd081485]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:29.589839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:29.591338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:29.593205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54efc156-32bf-43a1-859a-03158e46b784]: (4, ['qrouter-8f483531-0156-4109-94b6-07df8031167f', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.593970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:29.594315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:29.594568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:29.594901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:29.595974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1dc3aa5-1c1e-4a7b-8cb7-abd508e87303]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.597049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9424954-8692-43e7-b181-c40589e8a707]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.598573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af3afaa7-ba3c-44cb-9ef7-bf8b3e4619a6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.620924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f46bfb9f-83c9-40d5-8113-a804f855b1f2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 584560, 'reachable_time': 43000, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189837, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.623920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-8f483531-0156-4109-94b6-07df8031167f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:29.624315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c477a57b-bece-411b-be78-0e1f2d6654e6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:29.626330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-1253834789-router", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242" Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:22Z", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:9c:51", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4814, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:26Z", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e47dc9fc-8887-4123-86b9-b28f2b8be563", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "eea6a246-41a4-4c29-9aa5-cba81e33c4e5", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:35:73", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4819, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:21Z", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:24Z", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:29.629370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 80149f1b-838b-40a6-a304-75bb5e595cd7. Wait time elapsed: 6.372 Jul 01 10:22:29.643918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 80149f1b-838b-40a6-a304-75bb5e595cd7. Wait time elapsed: 6.373 Jul 01 10:22:29.643918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:31.278992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.960s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:31.280023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 06ce6427-dab9-46fd-8b57-1c3f490f0250 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:31.280314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:31.280609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:31.280964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:31.282127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d720fbb-d8f8-41ef-bfd8-b446c369e885]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.298694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a649fc5-9519-4bd3-a67e-4e465271e3e2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.320223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b43036c-f5bb-4e1b-88a2-1499f78839f1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.347201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2511a269-582a-4cb7-ace7-029e1004b102]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 189900, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.369658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ed3f475-c891-4a05-a895-8f8a05f54175]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.390794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbf3a9f2-a03f-42e6-9bc4-64b488bed034]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.413687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a157948-8638-477d-94bb-e85fcfc908fd]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.453645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de560dea-6296-4fdc-a4a2-424373451bf3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.454284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:31.454625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:31.454890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:31.455194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(46c82d52-0298-422e-871d-ddf9005b2611) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:31.470550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:31.470848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9917a99f-814c-4b42-89f3-8b568d371099]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.473472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[584eeea2-f42f-492e-8f17-58628eca2a78]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.474593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-46c82d52-02, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:31.505939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-46c82d52-02, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:31.506768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:31.507226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:31.507705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:31.508262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-46c82d52-02, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '46c82d52-0298-422e-871d-ddf9005b2611', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:22:41:9e'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:31.568766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10ac328c-9b58-47c4-ab8a-7a979ea5f2df]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.596778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[dc0dc9be-d127-4d83-91d4-99ca59e48a78]: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:31.596778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:31.596778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:31.596778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:31.596778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:31.596778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:31.596778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:31.596778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:31.596778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:31.596778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:31.596778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:31.596778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:31.596778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:31.597805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc0dc9be-d127-4d83-91d4-99ca59e48a78]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:31.598651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:31.600740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:31.603843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[131f5c0c-dd79-4c49-8bd8-e1d7b909a9d4]: (4, ['qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.604733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:31.605118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:31.605382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:31.605784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:31.606907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6262ba4-50c8-4ed4-a65a-7099189ce626]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.607940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b414b25d-a38a-4463-ad41-6d2923963a19]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.608955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0159b21-cd0f-44af-9c8e-1859239275da]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.628434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36b89d70-4e2e-48e7-bcb6-6ad9ff5a1a0d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 584760, 'reachable_time': 16288, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189918, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.634311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:31.635242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a056cfbd-628b-452d-b0f5-ad2b433ca5dc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:31.636609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-583633917-router", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:13Z", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:16Z", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:10:e3:5c", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4800, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:14Z", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:19Z", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "46c82d52-0298-422e-871d-ddf9005b2611", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "4e509fcf-4f66-429c-9cb9-2d53c43b2b37", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:22:41:9e", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4804, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:15Z", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:27Z", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:31.641114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id e5f26787-8340-4087-8d78-e2d95be65fe4. Wait time elapsed: 16.426 Jul 01 10:22:31.656276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id e5f26787-8340-4087-8d78-e2d95be65fe4. Wait time elapsed: 16.426 Jul 01 10:22:31.656276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:32.404255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.766s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:32.404865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 06ce6427-dab9-46fd-8b57-1c3f490f0250 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:32.405990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:32.406311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:32.406617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:32.407930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96e8f6d4-7220-4763-9650-720b0f108f90]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.425637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2a8c2d8-55ad-4b10-af64-cd9544c6b387]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.448713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f10472f3-e91e-4097-a1f6-a6d5d3225ab1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.477439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7f1516b-85be-4365-aa74-4b0f977add07]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 189921, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.500082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbb6679c-88b4-47d5-a9bc-1ab75679a9c1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.521001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[779afa5d-e729-4596-9623-fdc0540cc99c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.542302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81432981-9df4-468f-9686-dcacd92802b2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.562339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5dda91ed-38ee-4d3c-bb9c-cd2097eb906f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.562835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:32.563109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:32.563386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:32.563693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(46c82d52-0298-422e-871d-ddf9005b2611) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:32.579086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:32.579401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[000a9a89-64f2-4129-97d0-7e5a17c641d9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.582081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3983a0f4-6848-4c42-8bb0-64cb88c4952b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.583041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-46c82d52-02, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:32.617934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-46c82d52-02, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:32.620101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:32.620101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:32.620404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:32.621110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-46c82d52-02, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '46c82d52-0298-422e-871d-ddf9005b2611', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:22:41:9e'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:32.676893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1de8dbc0-8e15-4883-8978-d5805e84782b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.716167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ad9dd012-0587-4833-9746-6c47f9671036]: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:32.716167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:32.716167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:32.716167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:32.716167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:32.716167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:32.716167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:32.716167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:32.716167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:32.716167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:32.716167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:32.716167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:32.716167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:32.717347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad9dd012-0587-4833-9746-6c47f9671036]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:32.718379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:32.720723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:32.723575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdf434ff-4570-4df7-bef6-8999f4e0c7cd]: (4, ['qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.724477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:32.724932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:32.725633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:32.726054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:32.727094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32a2ee3e-7734-49a9-98d2-cf1b7af66ff8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.728149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d9df165-b271-485d-b7d8-1dd136cd4259]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.729174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ade60472-dfc7-4468-871a-e6e87184f97d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.748186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ad9826c-b2d3-44be-98cc-f1917758ed79]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 584874, 'reachable_time': 44948, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189942, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16433a86-5583-4eb7-b175-6ade24335aba]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-583633917-router", Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:32.753033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:13Z", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:16Z", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:10:e3:5c", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4800, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:14Z", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:19Z", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "46c82d52-0298-422e-871d-ddf9005b2611", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "4e509fcf-4f66-429c-9cb9-2d53c43b2b37", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:22:41:9e", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4804, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:15Z", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:29Z", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id db3e68ce-65c0-4eab-89ff-e34465f4eef7. Wait time elapsed: 14.909 Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id db3e68ce-65c0-4eab-89ff-e34465f4eef7. Wait time elapsed: 14.909 Jul 01 10:22:32.757538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:33.710826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.082s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:33.711424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 8f483531-0156-4109-94b6-07df8031167f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:33.712282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (8f483531-0156-4109-94b6-07df8031167f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:33.713300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:33.713672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:33.715021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[683179ba-ef10-4192-8010-91b599b9df9b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:33.728255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[422579ec-4e7a-407c-9dd6-683b24e465d4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:33.749411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b43b388b-d26b-4f94-a00d-64047d8650ad]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:33.785995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[571426b9-8bf0-465f-b732-541a3022519b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 189945, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:33.807996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aebf277e-dda4-41f7-9c56-ac4ffc3dab5d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:33.832260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b16ca959-7a8f-4770-8740-154a7fb4f6f6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:33.853500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbf40e4c-b3a2-42ac-a168-e80c3b166eb7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:33.874581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e15bfbba-27c8-4659-a35d-ec734532ab17]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:33.875158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:33.875469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:33.875838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:33.876126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e47dc9fc-8887-4123-86b9-b28f2b8be563) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:33.891201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:33.891430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67a34e7b-ddcf-41dc-821c-8c4b8b52205c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:33.894108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70988e79-b8ef-43a5-bab1-6e0d742f02b1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:33.895130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e47dc9fc-88, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:33.934219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e47dc9fc-88, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:33.934875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:33.935257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:33.935687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:33.936195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e47dc9fc-88, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e47dc9fc-8887-4123-86b9-b28f2b8be563', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:35:73'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:33.994324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f52e4151-17da-40a5-8e72-41659a3e0c23]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.025843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2ea9c67b-5412-4e66-9e30-d47e2389f20c]: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:34.025843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:34.025843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:34.025843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:34.025843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:34.025843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:34.025843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:34.025843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:34.025843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:34.025843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:34.025843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:34.025843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:34.025843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:34.026753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ea9c67b-5412-4e66-9e30-d47e2389f20c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:34.027176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:34.028743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:34.031057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a16646d6-e3f9-4d3c-b86a-786f1b3e52df]: (4, ['qrouter-8f483531-0156-4109-94b6-07df8031167f', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.031057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:34.031057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:34.031057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:34.031057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:34.032017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3404efa3-65a0-4137-8221-5e36716de2b2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.033178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05eb1aa6-a466-4378-acc5-efb528a471c9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.034143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bcb4739-4cce-4e38-b0b0-78900466eb19]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.059654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4a5cb8b-314e-441c-9aa1-2908b55dcaa9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 585004, 'reachable_time': 31824, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189964, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.063973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-8f483531-0156-4109-94b6-07df8031167f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:34.063973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fabbd0b-1b16-4df6-8516-bba2dea5a6c4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:34.064972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-1253834789-router", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242" Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:22Z", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:9c:51", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4814, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:26Z", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e47dc9fc-8887-4123-86b9-b28f2b8be563", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "eea6a246-41a4-4c29-9aa5-cba81e33c4e5", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:35:73", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4819, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:21Z", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:31Z", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:34.067818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 7ee424a3-69bd-4bf2-9a28-a0bbedf6e145. Wait time elapsed: 13.270 Jul 01 10:22:34.079469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 7ee424a3-69bd-4bf2-9a28-a0bbedf6e145. Wait time elapsed: 13.270 Jul 01 10:22:34.079469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:34.808247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.742s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:34.808862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 8f483531-0156-4109-94b6-07df8031167f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:34.809416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (8f483531-0156-4109-94b6-07df8031167f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:34.809734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:34.810013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:34.811196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c693344-bb5e-43e4-88be-f7c11c4130a1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.825923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12b16072-fb4c-41b3-9ca0-10b27f79134b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.847676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec3feb38-8a61-4d99-9771-ba91ba6d3c8e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.873035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e41d357-4fc0-4370-83bc-efa0e4c2d0fd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 189969, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.894765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8040c446-9577-4694-8f57-4b4d685561a2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.915530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c5d217c-3351-4211-969a-5f0ff540c361]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.936449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2b88cd2-9596-4fb5-b812-373b459d8462]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.957557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[355ab71d-b905-406d-a1a8-50b60f607909]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.958242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:34.958607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:34.958939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:34.959285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e47dc9fc-8887-4123-86b9-b28f2b8be563) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:34.975337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:34.975664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[753ccc36-5bd4-4f63-a04e-4770e9173ada]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.979141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a7c5670-9849-455c-a945-863371a987e6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:34.980254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e47dc9fc-88, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:35.014584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e47dc9fc-88, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:35.015536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:35.015933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:35.016322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:35.016701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e47dc9fc-88, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e47dc9fc-8887-4123-86b9-b28f2b8be563', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:35:73'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:35.077471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[436a25d0-5214-4890-b026-ebeba3cf7e70]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.101340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[fb3dbab7-67e0-48d3-a86c-c6624ca17bbe]: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:35.101340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:35.101340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:35.101340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:35.101340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:35.101340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:35.101340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:35.101340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:35.101340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:35.101340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:35.101340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:35.101340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:35.101340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:35.102362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb3dbab7-67e0-48d3-a86c-c6624ca17bbe]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:35.102543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:35.104900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:35.107108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d81d406e-df6c-4813-acc9-bcf9ebd5ec7d]: (4, ['qrouter-8f483531-0156-4109-94b6-07df8031167f', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.107108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:35.107108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:35.107108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:35.107108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:35.108063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44613a78-9b7a-4b97-be46-c8d6e18955dc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.109069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec885d99-5d9a-40ed-bcb2-f6dca00e29b1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.110206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d74fe769-3773-4365-a300-ee00df2bbefe]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.131693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acaaa602-c27f-4d09-a59a-be9e32433dce]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 585113, 'reachable_time': 37108, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 189987, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.138769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-8f483531-0156-4109-94b6-07df8031167f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:35.139211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed582572-6ffc-4559-bdc7-08a0c32d2d58]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:35.140561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-1253834789-router", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242" Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:22Z", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:9c:51", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4814, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:26Z", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e47dc9fc-8887-4123-86b9-b28f2b8be563", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "eea6a246-41a4-4c29-9aa5-cba81e33c4e5", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:35:73", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4819, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:21Z", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:32Z", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:35.144831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:35.155886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 80149f1b-838b-40a6-a304-75bb5e595cd7. Wait time elapsed: 11.890 Jul 01 10:22:35.155886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 80149f1b-838b-40a6-a304-75bb5e595cd7. Wait time elapsed: 11.890 Jul 01 10:22:35.155886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:35.635537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.882s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:35.635809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 06ce6427-dab9-46fd-8b57-1c3f490f0250 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:35.636731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:35.637053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:35.637510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:35.638712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9fb5da26-897a-4bec-8ea7-c2956ca9750c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.654769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f11cfff-f1d6-4f55-aa85-c471004b3770]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.683663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c270597-dbd8-4c88-a182-51aecef9da86]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.718033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dcb95736-1df9-4322-af6b-985d3974d020]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 189990, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.742734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67f281f9-d817-4ff9-a02e-94258d0c6ccd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.765531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[502d1c60-2c75-469c-92a7-26ef5bcd41f4]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.789537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3310ab6-9d5a-45ba-a8b2-a5da209df32b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.811401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93df8e7f-b680-4e0f-a1ae-3aaf64c98a63]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.812777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:35.813484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:35.813835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:35.814177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(46c82d52-0298-422e-871d-ddf9005b2611) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:35.831702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:35.831995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36c26499-ab49-494a-be2c-8f42cb2f208a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.834871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b5dbb17-588d-44f7-ac2f-aeb1f9160ed6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.835849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-46c82d52-02, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:35.869857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-46c82d52-02, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:35.870655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:35.871223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:35.871670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:35.872048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-46c82d52-02, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '46c82d52-0298-422e-871d-ddf9005b2611', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:22:41:9e'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:35.924996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[717186b9-e6d9-46e9-b5a8-212f6e4e602e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.963037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[acbc73ba-c278-4354-9b83-986a40f30b1c]: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:35.963037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:35.963037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:35.963037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:35.963037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:35.963037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:35.963037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:35.963037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:35.963037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:35.963037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:35.963037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:35.963037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:35.963037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:35.964368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acbc73ba-c278-4354-9b83-986a40f30b1c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:35.965441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:35.967110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:35.969122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3dd4beb-902c-4417-92b7-f82a9cdc9fce]: (4, ['qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.969817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:35.970134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:35.970383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:35.970721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:35.972251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0de8e974-103a-4c01-9a73-5e18e9c48f0a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.973292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6b9d817-3871-4031-b1ae-44e62d383d90]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.974352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec695275-eae0-453c-a624-c5cb076e0e31]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.993731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2d22631-63e4-4db2-92f4-eeddd37f89ac]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 585198, 'reachable_time': 29120, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 190008, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:35.999017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:35.999479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4a0e453-b642-4cbd-b582-60e7aeae4ba4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:36.000711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-583633917-router", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:13Z", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:16Z", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:10:e3:5c", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4800, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:14Z", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:19Z", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "46c82d52-0298-422e-871d-ddf9005b2611", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "4e509fcf-4f66-429c-9cb9-2d53c43b2b37", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:22:41:9e", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4804, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:15Z", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:33Z", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:36.003395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id e5f26787-8340-4087-8d78-e2d95be65fe4. Wait time elapsed: 20.790 Jul 01 10:22:36.014907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id e5f26787-8340-4087-8d78-e2d95be65fe4. Wait time elapsed: 20.791 Jul 01 10:22:36.014907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:36.805329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.802s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:36.805599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 06ce6427-dab9-46fd-8b57-1c3f490f0250 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:36.807130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:36.807470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:36.807751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:36.808914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9da90f1f-43cd-43d0-90f8-44198e4c8160]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:36.821595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2cac35c-cf76-47f1-922f-6e7293689373]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:36.843132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0dbc0ffd-f422-4b98-bcdc-d0ad62cef7d5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:36.886339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[233fe595-f67c-4d19-9b48-88842be38b86]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 190011, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:36.908562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71068b33-4a63-4ffe-94bf-617a7b70001a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:36.929936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2f8a4b8-965c-45d3-90ff-334480e77cbd]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:36.951962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88b8e3aa-f6a6-4776-a891-e92e5837d733]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:36.973442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[410b9796-7861-4ed9-9e59-28436ba01fb5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:36.974085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:36.974360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:36.974629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:36.975060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(46c82d52-0298-422e-871d-ddf9005b2611) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:36.990318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:36.990704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61741c22-8e52-4c0b-9195-fd71210a0fd6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:36.994780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eadc131f-35ce-456d-a6b6-2372c6414d45]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:36.995983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-46c82d52-02, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:37.047726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-46c82d52-02, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:37.048489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:37.049278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:37.049766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:37.050252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-46c82d52-02, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '46c82d52-0298-422e-871d-ddf9005b2611', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:22:41:9e'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:37.106564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45e8e607-4780-491c-8e37-d1c02ffa57d9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:37.142300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b5d76bd2-4091-468c-b17c-6e0ddd77c69f]: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:37.142300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:37.142300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:37.142300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:37.142300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:37.142300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:37.142300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:37.142300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:37.142300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:37.142300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:37.142300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:37.142300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:37.142300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:37.143294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5d76bd2-4091-468c-b17c-6e0ddd77c69f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:37.143609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:37.145054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:37.147415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3e98f79-afb5-4596-94bc-04d4c8036ea4]: (4, ['qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:37.147415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:37.147415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:37.147415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:37.147755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:37.148878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92720954-53ae-4256-b9e2-2f124e583ec2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:37.150128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56be8a23-273d-4a77-88ae-b4fdab8af51b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:37.151004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f79d2bd2-48de-4812-8918-dae0cea2e1bd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:37.171393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce1eb62d-e001-4aaf-b0f0-676275d9a52b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 585314, 'reachable_time': 18488, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 190032, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:37.178178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:37.178498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abd9a28a-67d9-4ac3-8ba8-dedbab3a4fec]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:37.179721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-583633917-router", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:13Z", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:16Z", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:10:e3:5c", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4800, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:14Z", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:19Z", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "46c82d52-0298-422e-871d-ddf9005b2611", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "4e509fcf-4f66-429c-9cb9-2d53c43b2b37", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:22:41:9e", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4804, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:15Z", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:33Z", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:37.184480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id db3e68ce-65c0-4eab-89ff-e34465f4eef7. Wait time elapsed: 19.337 Jul 01 10:22:37.195423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id db3e68ce-65c0-4eab-89ff-e34465f4eef7. Wait time elapsed: 19.338 Jul 01 10:22:37.195423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:39.166209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.019s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:39.166775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 8f483531-0156-4109-94b6-07df8031167f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:39.167305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (8f483531-0156-4109-94b6-07df8031167f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:39.167650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:39.167930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:39.169084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[103861e4-aa4f-4e8c-9ce8-4750f08d221c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.183127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c594759c-366a-4dcd-bb35-cfab1175b853]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.203763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9d7e51b-652b-4eb9-a6cf-65a82a6ea52c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.238395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60389f9b-a12c-4d66-86e9-4bb06f4cd98e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 190096, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.260806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3e75804-4939-43aa-acd7-1ada7e9de4eb]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.283818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[740d1f2c-53cf-4a89-af72-c986ff2aaca1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.311210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[254959a8-4cbb-4b99-a38f-57808a4786d8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.335961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cf8e18d-4596-4c75-a738-77b8da254791]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.336523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:39.336813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:39.337110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:39.337949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e47dc9fc-8887-4123-86b9-b28f2b8be563) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:39.353035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:39.353901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77abc231-9517-4535-a8b1-717ea7a44a9f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.356794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2655d14f-1a2e-4649-a889-8416184254c6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.358369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e47dc9fc-88, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:39.412911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e47dc9fc-88, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:39.413689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:39.414182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:39.414704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:39.415263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e47dc9fc-88, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e47dc9fc-8887-4123-86b9-b28f2b8be563', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:35:73'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:39.476430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6f51427-eefd-4ba6-b4a8-639930e99bcb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.514181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1e247802-9579-4aba-9a03-729d83f3812f]: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:39.514181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:39.514181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:39.514181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:39.514181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:39.514181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:39.514181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:39.514181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:39.514181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:39.514181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:39.514181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:39.514181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:39.514181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:39.515119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e247802-9579-4aba-9a03-729d83f3812f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:39.515457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:39.517947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:39.522489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[afa12dc7-1688-4226-a97d-34b23363cd6f]: (4, ['qrouter-8f483531-0156-4109-94b6-07df8031167f', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.523293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:39.523708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:39.523966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:39.524308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:39.526490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb97b44e-0baf-4ae1-97c3-f2fc1d571932]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.527723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[347b0bc5-8708-4f07-b93c-f2248268159a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.528748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6bc90315-00d1-43ab-a383-2c9e1da99e2e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.550036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a4dc2bf-a18f-41cf-abff-d14f1458b3de]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 585549, 'reachable_time': 41312, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 190114, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.552099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-8f483531-0156-4109-94b6-07df8031167f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:39.552683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71c94b57-a8af-4883-bc1d-116e43fd336a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:39.555726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-1253834789-router", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242" Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:22Z", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:9c:51", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4814, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:26Z", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e47dc9fc-8887-4123-86b9-b28f2b8be563", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "eea6a246-41a4-4c29-9aa5-cba81e33c4e5", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:35:73", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4819, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:21Z", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:32Z", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:39.559155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 7ee424a3-69bd-4bf2-9a28-a0bbedf6e145. Wait time elapsed: 18.761 Jul 01 10:22:39.574353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 7ee424a3-69bd-4bf2-9a28-a0bbedf6e145. Wait time elapsed: 18.761 Jul 01 10:22:39.574353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:40.338282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.779s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:40.338904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 8f483531-0156-4109-94b6-07df8031167f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:40.340301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (8f483531-0156-4109-94b6-07df8031167f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:40.340825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:40.341281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:40.342981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20212197-ae0a-425d-b256-f12771f28655]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.363883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d49da74-e934-4d81-9cc0-733821684ed0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.386281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32a29364-85ec-482f-b18b-4e3204cdca07]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.419270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6281334-ccf0-4079-9cd5-a0a1d3f9b3a5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 190117, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.440623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b880404a-a2bf-48d9-9595-55cd507c757a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.462938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0cd4b730-cdf9-4cb8-9342-917efe1573d2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.484609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a5f90a3-12bb-4256-88cf-ccb886033007]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.510212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6baf7757-371d-4e77-8278-61dc22f30c1e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.510927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:40.511303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:40.511715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:40.512090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e47dc9fc-8887-4123-86b9-b28f2b8be563) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:40.530764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:40.531997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[228a10d3-2b42-4053-b6c1-c67e97afc32b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.535333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3d5c9ac-21c7-430c-be0f-e6be94577723]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.536627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e47dc9fc-88, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:40.578471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e47dc9fc-88, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:40.579544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:40.580279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:40.580922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:40.581497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e47dc9fc-88, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e47dc9fc-8887-4123-86b9-b28f2b8be563', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:35:73'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:40.639538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c9051c2-3f7b-4663-aabe-f77be612a93e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.678181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[26752128-ba83-4c45-96ef-dcec2430b30a]: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:40.678181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:40.678181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:40.678181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:40.678181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:40.678181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:40.678181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:40.678181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:40.678181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:40.678181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:40.678181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:40.678181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:40.678181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:40.679744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26752128-ba83-4c45-96ef-dcec2430b30a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:40.680763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:40.683747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:40.689315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75768a73-a496-4f4e-9692-2b54561dcbc8]: (4, ['qrouter-8f483531-0156-4109-94b6-07df8031167f', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.690185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:40.690600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:40.690926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:40.691401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:40.692572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58c7a5d5-15d8-4a6a-bacc-7f8040410d08]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.693597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[311c8177-2ea9-4773-b166-47514e0a48c5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.694749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15076a4b-8a73-4c98-b24b-dd0e7e986226]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.716204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d71b88b0-df0c-4767-a313-eda5b37a0a6f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 585668, 'reachable_time': 24980, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 190138, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.724443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-8f483531-0156-4109-94b6-07df8031167f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:40.724792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[104f5a57-7dcc-4b7d-ace1-f15f088f5ed1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:40.726158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-1253834789-router", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242" Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:22Z", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:9c:51", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4814, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:26Z", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e47dc9fc-8887-4123-86b9-b28f2b8be563", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "eea6a246-41a4-4c29-9aa5-cba81e33c4e5", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:35:73", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4819, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:21Z", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:38Z", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:40.729689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:40.742355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 80149f1b-838b-40a6-a304-75bb5e595cd7. Wait time elapsed: 17.475 Jul 01 10:22:40.742355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 80149f1b-838b-40a6-a304-75bb5e595cd7. Wait time elapsed: 17.476 Jul 01 10:22:40.742355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:41.287454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.105s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:41.287722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 06ce6427-dab9-46fd-8b57-1c3f490f0250 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:41.289378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:41.289952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:41.291185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:41.292919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[850a7ab1-bef6-4cff-ba5c-0bf562a7ce1b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.310393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5761f51f-9ba6-4d10-97c7-2a1f6cdca652]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.337063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e270363-37e4-4948-b50b-a964d82ec130]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.379500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0812488a-7508-4739-bc3f-851fd36579ca]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 190171, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.403185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d326f93d-ba80-4699-960f-43780ab863bc]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.423944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[362d0bec-0d2b-4149-bf4b-4c6e8f1b59f6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.446522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5085977-83aa-4a4c-8fde-a380f4ce66fa]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.467559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a7029bd-a0a2-4f75-a694-d85d02d0b1b3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.468121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:41.468398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:41.468666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:41.468946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(46c82d52-0298-422e-871d-ddf9005b2611) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:41.484624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:41.484919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2692c230-6be4-4caf-b0e3-6d4b27b475f7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.487903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fa88225-1a15-41a7-9e3d-f440f62214bd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.489019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-46c82d52-02, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:41.523040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-46c82d52-02, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:41.524576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:41.525668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:41.526274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:41.526991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-46c82d52-02, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '46c82d52-0298-422e-871d-ddf9005b2611', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:22:41:9e'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:41.581013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69c7a1b4-c10a-4da6-84e6-6c97d4c39bdf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.607697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[acaf9f5b-5b87-4751-9da7-cf14a653fe30]: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:41.607697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:41.607697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:41.607697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:41.607697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:41.607697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:41.607697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:41.607697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:41.607697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:41.607697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:41.607697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:41.607697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:41.607697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:41.608710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acaf9f5b-5b87-4751-9da7-cf14a653fe30]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:41.608805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:41.610589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:41.614555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1080c575-7a44-40a6-9966-07fc3d0c5444]: (4, ['qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.615319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:41.615687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:41.615949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:41.616290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:41.617339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f9c4b4c-3a10-4f57-979a-4a74d840f07e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.618317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[797293b3-4ab3-46b2-b6c0-14112cf1e96e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.619478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0fe1c16-1d26-4dfc-8a42-62966cf713ed]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.640896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53bc02d6-deaf-43b0-a148-44a41207a293]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 585764, 'reachable_time': 18996, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 190189, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.648076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:41.648445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae1d564c-31d3-4f3d-b34f-c110a4d440da]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:41.649421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-583633917-router", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:13Z", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:16Z", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:10:e3:5c", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4800, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:14Z", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:19Z", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "46c82d52-0298-422e-871d-ddf9005b2611", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "4e509fcf-4f66-429c-9cb9-2d53c43b2b37", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:22:41:9e", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4804, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:15Z", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:36Z", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:41.653985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id e5f26787-8340-4087-8d78-e2d95be65fe4. Wait time elapsed: 26.439 Jul 01 10:22:41.667855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id e5f26787-8340-4087-8d78-e2d95be65fe4. Wait time elapsed: 26.439 Jul 01 10:22:41.667855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:42.511906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.860s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:42.512563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 06ce6427-dab9-46fd-8b57-1c3f490f0250 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:42.513116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:42.513441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:42.513718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:42.514940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03920f4d-b7d5-4e17-acc5-fad360b69260]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.533287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c52811c-6749-4fe8-a3a9-01fdd6ed076d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.560444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0eab2f7-be82-462f-b408-213b8f7fb74a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.600910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e45e961-87ac-40f0-9cd3-da71563f4280]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 190193, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.623701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f432fae4-7ef9-4ada-b719-436db8c520fe]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.654422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[347895d0-dc89-4246-8d19-065fce293f5e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.679191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95136c97-4417-4037-b5bb-73c6473d8e19]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.702896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c2da662-5771-4a67-bac9-62e08cbb7acb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.704707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:42.705493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:42.705998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:42.706495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(46c82d52-0298-422e-871d-ddf9005b2611) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:42.725018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:42.725616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8601c550-b482-40dc-a0af-489ad079f68d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.728701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32ead838-1636-41fc-8f4f-02ef7acc9d22]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.729935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-46c82d52-02, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:42.770532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-46c82d52-02, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:42.771776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:42.772500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:42.773231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:42.773823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-46c82d52-02, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '46c82d52-0298-422e-871d-ddf9005b2611', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:22:41:9e'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:42.854206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05e7b36f-41e1-43ba-a85b-f1ddba08af25]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.896713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[59045e3e-9465-4a05-80e0-eb04511f7441]: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:42.896713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:42.896713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:42.896713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:42.896713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:42.896713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:42.896713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:42.896713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:42.896713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:42.896713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:42.896713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:42.896713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:42.896713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59045e3e-9465-4a05-80e0-eb04511f7441]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:42.898913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:42.907428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:42.907428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:42.907428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:42.907428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:42.907428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:42.907428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:42.907428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:42.907428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:42.907428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:42.907428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:42.907428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:42.907428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:42.910022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e20d286a-5e48-4859-a1a3-be10da259280]: (4, ['qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.910863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:42.911236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:42.911523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:42.911869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:42.912985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9f730d1-b7a2-4620-8e92-c68adee80c4a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.913939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d81d4db-3c1e-4308-a5b7-0ba310b330f6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.914986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[038af015-6a1a-49d3-a148-3189c6fdebb6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.941559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4160e6e5-b9de-4eb9-9d3e-75fa59b936c2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 585886, 'reachable_time': 32232, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 190211, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.943558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:42.944084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[adc19244-889a-4d4b-8449-728da5d94f9e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:42.945302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-583633917-router", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:13Z", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:16Z", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:10:e3:5c", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4800, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:14Z", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:19Z", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "46c82d52-0298-422e-871d-ddf9005b2611", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "4e509fcf-4f66-429c-9cb9-2d53c43b2b37", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:22:41:9e", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4804, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:15Z", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:39Z", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:42.950473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:42.963182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3 Jul 01 10:22:42.963182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id db3e68ce-65c0-4eab-89ff-e34465f4eef7. Wait time elapsed: 25.107 Jul 01 10:22:42.963182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id db3e68ce-65c0-4eab-89ff-e34465f4eef7. Wait time elapsed: 25.107 Jul 01 10:22:42.963182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:43.724874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.991s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:43.725817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 8f483531-0156-4109-94b6-07df8031167f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:43.727342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (8f483531-0156-4109-94b6-07df8031167f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:43.728766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:43.729212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:43.733443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2bf773b-60d8-4d43-b97e-36eded02871f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:43.748877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a2b5b92-a032-45b7-a731-4c7ee79d066c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:43.771756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7527a3e-7b96-466e-b3e9-8f7452e8d9f2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:43.802486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6833fae-8757-4bbb-9fa9-1f15af49f841]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 190218, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:43.824680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[678cb498-3038-4f19-b2d2-d59fbbbddfad]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:43.846019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47309487-e521-4aec-936c-0042da025d51]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:43.858756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ccb6afc-b1fb-4b82-99ae-0054ff2b8382]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:43.880885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[827b1dbb-731e-4dfb-b311-c3cade20005f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:43.881485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:43.881759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:43.882023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:43.882298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e47dc9fc-8887-4123-86b9-b28f2b8be563) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:43.897375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:43.897657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4639190-6b9f-4db3-80a5-7cd5a12855ad]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:43.901209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6dd6d88-40f8-4de4-b190-76888e627656]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:43.902338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e47dc9fc-88, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:43.939159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e47dc9fc-88, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:43.940127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:43.940683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:43.941228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:43.942018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e47dc9fc-88, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e47dc9fc-8887-4123-86b9-b28f2b8be563', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:35:73'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:44.010801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0d99168-e6d7-4568-8f71-fff95440211a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.039871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[51ec6b7d-cad9-4f7b-9eb9-f7a2c900d16c]: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:44.039871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:44.039871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:44.039871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:44.039871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:44.039871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:44.039871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:44.039871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:44.039871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:44.039871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:44.039871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:44.039871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:44.039871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:44.042770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51ec6b7d-cad9-4f7b-9eb9-f7a2c900d16c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:44.043770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:44.045321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:44.047293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b942448-5dfd-488d-b28c-b9b9da6b4630]: (4, ['qrouter-8f483531-0156-4109-94b6-07df8031167f', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.047293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:44.047293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:44.047293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:44.047670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:44.049267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4f21034-5f39-4e58-8994-5b06d7d925e0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.050247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b00b572-ac88-43b4-89ed-636ff8cb4f09]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.051332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a0036d9-4db8-4189-907c-66c39e82b78b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.077955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2d56bd2-e59a-4354-8228-b16c506b551b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 586006, 'reachable_time': 39988, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 190240, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.084237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-8f483531-0156-4109-94b6-07df8031167f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:44.084711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8a23bf0-98ce-434d-ae9a-28b2f7ed27e6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:44.085948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-1253834789-router", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242" Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:22Z", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:9c:51", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4814, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:26Z", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e47dc9fc-8887-4123-86b9-b28f2b8be563", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "eea6a246-41a4-4c29-9aa5-cba81e33c4e5", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:35:73", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4819, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:21Z", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:40Z", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:44.091625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:44.110296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 7ee424a3-69bd-4bf2-9a28-a0bbedf6e145. Wait time elapsed: 23.304 Jul 01 10:22:44.110296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 7ee424a3-69bd-4bf2-9a28-a0bbedf6e145. Wait time elapsed: 23.305 Jul 01 10:22:44.110296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:44.832192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.731s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:44.832832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 8f483531-0156-4109-94b6-07df8031167f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:44.833595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (8f483531-0156-4109-94b6-07df8031167f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:44.833989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:44.834629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:44.836065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1286a152-e496-46f7-a847-d9a6b08b5f5a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.850039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81b02d88-6cd6-402f-9710-14f1abe39f55]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.874627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8fac9c1c-b901-4904-8e92-57114a9e2dcf]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.904308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0bf55468-9914-430f-9eff-ec207a1dfdc4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 190249, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.926386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[036ac1ed-6fc2-4f53-91f6-ec567e3ec406]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.947668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[febaa23b-9fab-4638-a1e6-461155b0d837]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.969277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee7f0cdc-efda-41f1-9987-7035806c7d61]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.991508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aaaf2063-ec1c-4658-919a-987eb04a7b5e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:44.992069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:44.992342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:44.992982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:44.992982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e47dc9fc-8887-4123-86b9-b28f2b8be563) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:45.011444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:45.012173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e9ede49-90ad-420c-bf8e-56c97a79db6a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:45.015149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae602118-18b7-44c4-9506-bbaf663f4deb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:45.016249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e47dc9fc-88, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:45.062512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e47dc9fc-88, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:45.064003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:45.064591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:45.065817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:45.066919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e47dc9fc-88, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e47dc9fc-8887-4123-86b9-b28f2b8be563', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:35:73'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:45.122513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecdf6125-3b93-4a7c-8617-785bb3027eac]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:45.169159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7ae943e2-1915-45a4-a97d-fd24bbff30ae]: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:45.169159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:45.169159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:45.169159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:45.169159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:45.169159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:45.169159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:45.169159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:45.169159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:45.169159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:45.169159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:45.169159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:45.169159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:45.170251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ae943e2-1915-45a4-a97d-fd24bbff30ae]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:45.171015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:45.172970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:45.175674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5eac1f08-33cb-40cf-b19a-41d8bc65313e]: (4, ['qrouter-8f483531-0156-4109-94b6-07df8031167f', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:45.176520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:45.176905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:45.177174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:45.177837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:45.179174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3a9ea2c-2eb2-41c3-9157-031b1ac4ac44]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:45.180133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e77cd4e9-2dbf-46db-96da-ec31cebb7bde]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:45.181994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f2e6f96-bd7a-4b9b-ac2f-58dde9c7e484]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:45.203250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecd38c7a-4fdf-4045-9689-758f1d6922d6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 586116, 'reachable_time': 24772, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 190269, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:45.211634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-8f483531-0156-4109-94b6-07df8031167f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:45.211895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d383148e-ab99-457a-a5b6-beadc07fc4f1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:45.213593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-1253834789-router", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242" Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:22Z", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:9c:51", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4814, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:26Z", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e47dc9fc-8887-4123-86b9-b28f2b8be563", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "eea6a246-41a4-4c29-9aa5-cba81e33c4e5", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:35:73", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4819, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:21Z", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:40Z", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:45.216175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 8f483531-0156-4109-94b6-07df8031167f, action 3 Jul 01 10:22:45.228026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 80149f1b-838b-40a6-a304-75bb5e595cd7. Wait time elapsed: 21.959 Jul 01 10:22:45.228026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 80149f1b-838b-40a6-a304-75bb5e595cd7. Wait time elapsed: 21.960 Jul 01 10:22:45.228026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:46.985840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.033s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:46.987301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 06ce6427-dab9-46fd-8b57-1c3f490f0250 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:46.988258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:46.988586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:46.988868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:46.991787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9bd5bab-d12a-4f0a-b2ab-e0f62a5f9847]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.005442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40de763f-9c1b-47c4-92ff-a984f00d9679]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.026891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b02a7bba-a5e1-49cb-8140-2d534af72887]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.056712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a15c3991-5f35-4bbb-a38a-ae3d25ad3094]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 190344, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.078403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[998830d7-ff7b-4a74-a571-5d98b6a873da]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.099958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9dfcfd91-615b-48a9-82d9-765e05e201b9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.120925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cecbb371-fdbc-4bf2-8875-e8aa582ff370]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.142175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cdf7fe0-1ae7-48b4-85c5-5298b79db24c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.142778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:47.143410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:47.143757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:47.144096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(46c82d52-0298-422e-871d-ddf9005b2611) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:47.161080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:47.161367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac5e2da9-2ff0-4234-a228-229646cb0382]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.164220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76b1ec00-0658-426b-a58e-731f36a25134]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.165789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-46c82d52-02, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:47.208239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-46c82d52-02, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:47.209199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:47.210882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:47.212322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:47.213511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-46c82d52-02, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '46c82d52-0298-422e-871d-ddf9005b2611', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:22:41:9e'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:47.267502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7aed6215-af8f-440d-8060-c453dbdd930b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.304518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[24d04e0f-9278-4f2f-bc73-27ae240f1e63]: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:47.304518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:47.304518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:47.304518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:47.304518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:47.304518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:47.304518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:47.304518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:47.304518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:47.304518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:47.304518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:47.304518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:47.304518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:47.305574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24d04e0f-9278-4f2f-bc73-27ae240f1e63]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:47.305712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:47.307644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:47.309971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b1ec0e6-39f2-45b9-805d-0aeffbc4afb3]: (4, ['qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.309971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:47.309971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:47.309971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:47.310334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:47.311308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d0d17a6-01b8-440c-83e6-489ca235138a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.312298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[befa5ecb-884c-4497-aba2-bb4db3e93c86]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.313231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc5c7d37-e42d-4feb-a474-6481ebb26844]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.335866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a1819f3-9460-4734-bd90-a1165f8648e2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 586332, 'reachable_time': 33128, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 190365, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.339818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:47.340350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16b158de-03da-4b4e-949d-881d4b6a40fb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:47.342755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-583633917-router", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:13Z", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:16Z", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:10:e3:5c", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4800, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:14Z", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:19Z", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "46c82d52-0298-422e-871d-ddf9005b2611", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "4e509fcf-4f66-429c-9cb9-2d53c43b2b37", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:22:41:9e", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4804, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:15Z", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:43Z", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:47.347266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:47.359641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id db3e68ce-65c0-4eab-89ff-e34465f4eef7. Wait time elapsed: 29.515 Jul 01 10:22:47.359641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 2, update_id db3e68ce-65c0-4eab-89ff-e34465f4eef7. Wait time elapsed: 29.515 Jul 01 10:22:47.359641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:48.153417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.794s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:48.154806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 06ce6427-dab9-46fd-8b57-1c3f490f0250 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:48.155882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:48.156424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:48.156822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:48.158296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0116307f-080c-44aa-8831-c05c4b6c9d26]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.173560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae7223fb-e6c3-4c2d-8e1a-82aa5868cc06]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.196708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d248d763-ed8b-4f2a-9a6b-86a26467ff23]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.230123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49b7806c-ce12-4648-a0d0-496a365e8299]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 190373, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.252436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b73c78e-8e87-4dd5-94ab-63b952ede1d1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.278586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[748cc704-93a8-46c2-a7ae-b3dcdba6d2cb]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.313574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27d2e179-6a1f-4834-8944-f3ae211bd587]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.337276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59826b81-698e-43c5-8408-33b04b52edf0]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.337877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:48.338158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:48.338429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:48.338887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(46c82d52-0298-422e-871d-ddf9005b2611) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:48.356532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:48.356939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97cefdbe-d427-4f81-917d-f8b95f04d616]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.359990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa7ac8a2-891a-4930-b188-0cebc5e2b89b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.360971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-46c82d52-02, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:48.408081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-46c82d52-02, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:48.410299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:48.410727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:48.411128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-46c82d52-02, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:48.411563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-46c82d52-02, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '46c82d52-0298-422e-871d-ddf9005b2611', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:22:41:9e'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:48.476565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3f9e1ce-2da2-407c-8355-c2f2448c671a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f3bf6685-b939-49ff-9aae-2d058b26de00]: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:48.515928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3bf6685-b939-49ff-9aae-2d058b26de00]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:48.518831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:48.523937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:48.523937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:48.523937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:48.523937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:48.523937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:48.523937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:48.523937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:48.523937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:48.523937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec0084db-7f37-4c99-81d9-ba053b5477f4]: (4, ['qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.523937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:48.523937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:48.523937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:48.523937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:48.526395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac39430d-b000-4fcf-8ab7-e6aadddcfd0b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.527408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a56fc59-fad3-431f-86f9-a698eed5b67e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.528354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc251f50-4bc1-4e84-82bb-56b02741f69c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.548816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4321b250-b152-4644-8270-2bb459873733]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 586449, 'reachable_time': 33680, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 190392, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.554093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:48.554093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[321b8ed9-126b-4317-b9f8-5486a457c204]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 06ce6427-dab9-46fd-8b57-1c3f490f0250: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-46c82d52-02 not found in namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250. Jul 01 10:22:48.556666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-583633917-router", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9" Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:13Z", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:16Z", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6e424432-aa98-4b1f-bfdc-786980c6bd5f", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:10:e3:5c", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4800, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.9", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:14Z", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:19Z", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "46c82d52-0298-422e-871d-ddf9005b2611", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "4e509fcf-4f66-429c-9cb9-2d53c43b2b37", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:22:41:9e", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "06ce6427-dab9-46fd-8b57-1c3f490f0250", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4804, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:15Z", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:46Z", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "40ea21452d79439aa9f09c5bcda8b49d", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b2a46d1b-0576-4d80-ae58-70cecffb6701", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:48.559567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3 Jul 01 10:22:49.195681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.980s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:49.196265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 8f483531-0156-4109-94b6-07df8031167f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:49.197693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (8f483531-0156-4109-94b6-07df8031167f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:49.198080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:49.198435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:49.199614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2d7f242-74b6-4fe5-b0fa-ca5b18968cf3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.224624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ada95d89-c900-43c8-9f0e-888c35aa9663]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.245992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b130c302-747c-487c-93c6-20d7ab7bdeaf]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.281444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f4f289d-30b3-42a0-8bb8-8b34cebe2747]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 190398, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.302843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8ecd97f-7fa1-4298-847f-c45029987598]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.323744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3b8445b-0531-4d82-b8df-b55d62a0968c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.344207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64e1187f-6926-4bc0-9462-7edb54e4ac11]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.365146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9671824-d253-4c6e-abb3-98be22675953]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.365731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:49.366006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:49.366269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:49.366546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e47dc9fc-8887-4123-86b9-b28f2b8be563) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:49.382842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:49.383138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[800a0652-2c35-4408-9401-19c251b9e3e4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.386498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0e08578-4e52-4227-897e-ed687bcf3d16]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.387613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e47dc9fc-88, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:49.436872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e47dc9fc-88, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:49.437533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:49.437922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:49.438306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:49.438865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e47dc9fc-88, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e47dc9fc-8887-4123-86b9-b28f2b8be563', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:35:73'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:49.481502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53b21f3d-5db0-4429-bf82-d448c6c04f97]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.510166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3fd10c39-6258-4171-8b3a-6e4141e7ce43]: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:49.510166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:49.510166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:49.510166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:49.510166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:49.510166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:49.510166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:49.510166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:49.510166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:49.510166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:49.510166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:49.510166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:49.510166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:49.511108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fd10c39-6258-4171-8b3a-6e4141e7ce43]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:49.511447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:49.513340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:49.515769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57227b68-cf31-44d8-b062-a67e36279793]: (4, ['qrouter-8f483531-0156-4109-94b6-07df8031167f', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.515769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:49.516184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:49.516327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:49.516604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:49.517637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[795d5a83-de5b-42a3-aeea-bc0859fca3c8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.518693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd66d0bb-5e0c-473d-8c81-1fbd0e555703]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.520238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b8efb24-485d-4375-a6d3-cb96255c87ea]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.542613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01e0e13b-2635-4539-9e30-92e51d65a194]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 586554, 'reachable_time': 21744, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 190416, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-8f483531-0156-4109-94b6-07df8031167f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e3b8edf-70ae-4a84-9dd9-6643cbe623f1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-1253834789-router", Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242" Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:49.547043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:22Z", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:9c:51", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4814, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:26Z", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e47dc9fc-8887-4123-86b9-b28f2b8be563", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "eea6a246-41a4-4c29-9aa5-cba81e33c4e5", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:35:73", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4819, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:21Z", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:44Z", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 80149f1b-838b-40a6-a304-75bb5e595cd7. Wait time elapsed: 26.292 Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 2, update_id 80149f1b-838b-40a6-a304-75bb5e595cd7. Wait time elapsed: 26.292 Jul 01 10:22:49.551121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:22:52.840060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.292s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:22:52.841628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 8f483531-0156-4109-94b6-07df8031167f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:22:52.843498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (8f483531-0156-4109-94b6-07df8031167f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:22:52.844044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:52.845164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:52.847089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c189efa4-3cd0-40bd-9cdc-54858fa72cbc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:52.879547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9588d2f7-a1f2-4f6f-95b1-ecbb29218b5e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:52.920953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf5140bd-5ffa-4b80-9e6d-addc81c73cff]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:52.966335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3a0c1ce-46ec-45dd-be98-326951520521]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 190433, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:52.997269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[179d93b7-bbcf-4481-9124-43d2b351ca47]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.068036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e760d6d-6517-4832-9000-89a20af37aac]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.107832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96238a24-8376-4342-a4c1-72d8c16d02c5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.137326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f14cb31-e748-4eb9-9025-3fa922c25964]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.138036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:22:53.138406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:22:53.138861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:22:53.139235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(e47dc9fc-8887-4123-86b9-b28f2b8be563) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:22:53.167498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:22:53.168121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd5b7410-ef02-4c6f-98ef-514718c96023]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.172488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03aae9e8-798b-4121-84b3-04e0b22f347d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.173679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-e47dc9fc-88, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:53.232676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-e47dc9fc-88, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:53.233446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:53.233779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:53.234176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-e47dc9fc-88, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:53.234574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-e47dc9fc-88, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'e47dc9fc-8887-4123-86b9-b28f2b8be563', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9c:35:73'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:22:53.305826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5b2c7cd-19b7-4a59-ae9f-9053536b75b8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.355255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6b8f288c-be6b-4b36-999c-e44710be9fbc]: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:22:53.355255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:22:53.355255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:22:53.355255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:22:53.355255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:22:53.355255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:22:53.355255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:22:53.355255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:22:53.355255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:22:53.355255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:22:53.355255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:22:53.355255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:22:53.355255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:53.356473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b8f288c-be6b-4b36-999c-e44710be9fbc]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:53.356571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:53.358379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:53.362975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f07fc923-1f06-49ae-9498-26e66f1231a4]: (4, ['qrouter-8f483531-0156-4109-94b6-07df8031167f', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.363799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:53.364266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:53.364663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:22:53.365275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/8f483531-0156-4109-94b6-07df8031167f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:22:53.368189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83617c5f-8207-4bc9-a1a2-87ace6bfa685]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.369522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f2d06bc-72e8-4d57-8535-a0db74f40636]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.371320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6642991b-c614-43e5-89e6-0e4303c02376]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.404368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47704fef-5906-4889-acad-de94864af8ff]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 586922, 'reachable_time': 30616, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 190499, 'error': None, 'target': 'qrouter-8f483531-0156-4109-94b6-07df8031167f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-8f483531-0156-4109-94b6-07df8031167f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d5fbd76-a92e-4dab-94ec-e0000194b200]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 8f483531-0156-4109-94b6-07df8031167f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-e47dc9fc-88 not found in namespace qrouter-8f483531-0156-4109-94b6-07df8031167f. Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestEncryptedCinderVolumes-1253834789-router", Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242" Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:53.409207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:22Z", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "29417014-056d-4f90-980f-1945d1a5e51e", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:9c:51", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4814, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.242", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:19Z", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:26Z", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e47dc9fc-8887-4123-86b9-b28f2b8be563", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "eea6a246-41a4-4c29-9aa5-cba81e33c4e5", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9c:35:73", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8f483531-0156-4109-94b6-07df8031167f", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4819, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:22:21Z", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:22:50Z", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fbb7bf8b3a6841ff990fd1117614cb97", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6908e33c-c9af-491e-8dde-e87e9a8e8579", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:22:53.414289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 8f483531-0156-4109-94b6-07df8031167f, action 3 Jul 01 10:22:55.262001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:23:15.140931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:23:15.141904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:23:15.142553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:23:15.142967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:23:35.263180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:23:52.593249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-cae2e69d-624a-4af4-a2cd-0da4b7a33764 admin admin] Got routers updated notification :['06ce6427-dab9-46fd-8b57-1c3f490f0250'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:23:52.593249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 1, update_id 0cfce6a4-3eaa-42a8-844f-7288f3c6c645. Wait time elapsed: 0.000 Jul 01 10:23:52.593249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 3, priority 1, update_id 0cfce6a4-3eaa-42a8-844f-7288f3c6c645. Wait time elapsed: 0.001 Jul 01 10:23:52.593249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:23:53.431480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.840s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:23:53.432126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 06ce6427-dab9-46fd-8b57-1c3f490f0250 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:23:53.434283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:23:53.437584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:23:53.438038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.003s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:23:53.439343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa1a0129-909f-456a-b688-31ab1f82618c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:53.454312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cc67685-88fa-4110-adf9-bf95477c4738]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:53.479083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[786d5ad6-7486-4f8c-b956-2baf8073db2e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:53.513377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0dc75447-f480-4f67-aaa2-cdaa6c9cfdcb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 191594, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:53.538805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[199eb37a-91bb-454a-b0bc-625ac588340c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:53.581771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e906f3f2-7506-457c-8bdf-2d2ed648f26d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:53.610551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d41a5363-48af-40e6-8fe9-6e14e3c86762]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:53.636300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4986783-ad20-41c9-8a99-2af492563cb1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:53.637199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:23:53.637675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:23:53.641008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:23:53.663471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d564b36-fcc0-450f-a011-8d7d5c2b3283]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 592977, 'reachable_time': 22940, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 191601, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:53.664237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:23:53.664605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:23:53.665619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:23:53.665874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:23:53.686502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d495b76d-09f7-48de-a1c6-78b3325e568a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 592977, 'reachable_time': 22940, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 191602, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:53.687314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:23:53.688205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:23:53.704526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65330b38-e099-48df-800b-1fa4c3786f53]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:53.751603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-4c8539f5-bec7-4229-9a8d-d0612fccb4e5 admin admin] Got router deleted notification for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:23:53.931257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78726287-138f-471a-85b9-92e6aec4eeb1]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:53.943190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd0e9db7-3772-483f-b506-0ca146c9b6fc]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.126226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92f0d630-20a5-4faf-b88a-53fde34f7a23]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:23:54.127574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:23:54.127859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:23:54.128140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" released by "process_external" :: held 0.462s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:23:54.128757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:23:54.129009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:23:54.129382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:23:54.129831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:23:54.154641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08977ad8-3135-40c8-bcdf-2c0cbc8b6f75]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.195631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e14d2f7-46d0-45a7-bb89-78d2d113f30a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.211265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ce1b7c2-d1ca-49b5-b06f-63f5b982eba4]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.256746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de9b0e2e-49c4-4517-b3fb-0a7ba6f53002]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.257383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:23:54.257761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:23:54.258093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:23:54.258482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" released by "process_address_scope" :: held 0.129s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:23:54.258949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:23:54.259528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:23:54.259945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:23:54.260402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:23:54.260914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:23:54.294248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[765767b8-6b78-4963-9ade-9f26e74e50ec]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.367199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f60ba5d8-2047-40bd-99df-ad77e6299185]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.381779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80a2c8fc-b192-4a10-b583-86e8a6b413af]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:23:54 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 10:23:54 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.447234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d7cbdbf-ecb5-46cc-8a8a-f9736fc20d2f]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.447920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:23:54.448234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:23:54.448497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:23:54.448780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250" released by "apply_metadata_nat_rules" :: held 0.189s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:23:54.449543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-06ce6427-dab9-46fd-8b57-1c3f490f0250 Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.haproxy Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 06ce6427-dab9-46fd-8b57-1c3f490f0250 Jul 01 10:23:54.450214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 10:23:54.453494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/06ce6427-dab9-46fd-8b57-1c3f490f0250.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 10:23:54.485831 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-06ce6427-dab9-46fd-8b57-1c3f490f0250[191616]: Proxy listener started. Jul 01 10:23:54.493832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 06ce6427-dab9-46fd-8b57-1c3f490f0250 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 10:23:54.494086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, update_id 0cfce6a4-3eaa-42a8-844f-7288f3c6c645. Time elapsed: 1.903 Jul 01 10:23:54.494560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 1, priority 1, update_id 49e1c3a7-ed26-4ee9-9e3b-a42f9cae3233. Wait time elapsed: 0.742 Jul 01 10:23:54.494825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 06ce6427-dab9-46fd-8b57-1c3f490f0250, action 1, priority 1, update_id 49e1c3a7-ed26-4ee9-9e3b-a42f9cae3233. Wait time elapsed: 0.743 Jul 01 10:23:54.495125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 06ce6427-dab9-46fd-8b57-1c3f490f0250 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 10:23:54.495476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:23:54.511760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[452bfbe0-e8e9-4458-a44e-13b314fc1bd0]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.513733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c687a9ba-fcf6-4ac9-9d5e-9f2e408146ed]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.514945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c381c42-2816-4ede-94f9-f71a8835f9d8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.515464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 10:23:54.516247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[810f34c4-f782-4800-9169-9000b30ff08a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.535682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9924b734-9a00-4ef9-be6e-22efc0794a99]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 592977, 'reachable_time': 22940, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 191622, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.537123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:23:54.537235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:23:54.559034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2261988d-38ea-45c0-b049-b09d2bb0e8ea]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 592977, 'reachable_time': 22940, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 191623, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.559817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 10:23:54.560353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:23:54.560682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/06ce6427-dab9-46fd-8b57-1c3f490f0250.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:23:54.560939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:23:54.561774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 06ce6427-dab9-46fd-8b57-1c3f490f0250 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 10:23:54.563044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[772791a2-2e57-4cdd-afe9-e912d60c262f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.588027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8803c00f-d996-49a2-b358-d0779f9e1b01]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 592977, 'reachable_time': 22940, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 191624, 'error': None, 'target': 'qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.597907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-06ce6427-dab9-46fd-8b57-1c3f490f0250 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:23:54.598312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3be7df16-e704-4e31-a43f-e2de33c5396f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:54.598840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 06ce6427-dab9-46fd-8b57-1c3f490f0250 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 10:23:54.599183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (06ce6427-dab9-46fd-8b57-1c3f490f0250), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:23:54.599778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:23:54.599990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:23:54.600474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:23:54.600784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:23:54.601178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 06ce6427-dab9-46fd-8b57-1c3f490f0250, update_id 49e1c3a7-ed26-4ee9-9e3b-a42f9cae3233. Time elapsed: 0.107 Jul 01 10:23:56.272482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-9a6414c4-7dfd-4af6-a758-041693825165 admin admin] Got routers updated notification :['8f483531-0156-4109-94b6-07df8031167f'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:23:56.274069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 1, update_id 8a1aaeb3-1b9e-4259-a565-24fdd16d4b13. Wait time elapsed: 0.001 Jul 01 10:23:56.274891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 3, priority 1, update_id 8a1aaeb3-1b9e-4259-a565-24fdd16d4b13. Wait time elapsed: 0.001 Jul 01 10:23:56.275194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:23:57.222500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-a3387b09-fb99-451d-a3f3-87c297a57486 admin admin] Got router deleted notification for 8f483531-0156-4109-94b6-07df8031167f {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:23:57.275462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.000s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:23:57.276070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 8f483531-0156-4109-94b6-07df8031167f was not found. Performing router cleanup Jul 01 10:23:57.277019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[545f987f-c698-48a1-9b89-7f70bc976e64]: (4, ['qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:57.277841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:23:57.278351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:23:57.278790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 8f483531-0156-4109-94b6-07df8031167f, update_id 8a1aaeb3-1b9e-4259-a565-24fdd16d4b13. Time elapsed: 1.005 Jul 01 10:23:57.279218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 8f483531-0156-4109-94b6-07df8031167f, action 1, priority 1, update_id 5e896c69-d2fc-4127-8fbd-196d4e5b9557. Wait time elapsed: 0.056 Jul 01 10:23:57.279592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 8f483531-0156-4109-94b6-07df8031167f, action 1, priority 1, update_id 5e896c69-d2fc-4127-8fbd-196d4e5b9557. Wait time elapsed: 0.056 Jul 01 10:23:57.280058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 8f483531-0156-4109-94b6-07df8031167f was not found. Performing router cleanup Jul 01 10:23:57.281225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b992d78-6a60-401f-866a-701c4089d11c]: (4, ['qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:23:57.281881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:23:57.282233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:23:57.282571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 8f483531-0156-4109-94b6-07df8031167f, update_id 5e896c69-d2fc-4127-8fbd-196d4e5b9557. Time elapsed: 0.003 Jul 01 10:24:02.960032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-80afad7f-7509-4291-8d0f-63e989b247ed admin admin] Got routers updated notification :['88bbc4fa-c162-4343-9551-ceded0fe88c3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:24:02.961605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id 4f3c4b9f-05b0-4205-ab6f-68576a2b0f73. Wait time elapsed: 0.001 Jul 01 10:24:02.961760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id 4f3c4b9f-05b0-4205-ab6f-68576a2b0f73. Wait time elapsed: 0.001 Jul 01 10:24:02.962032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:04.285412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.323s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:04.286392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:04.288006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:04.288457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:04.288743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:04.291634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10c2dd4f-595d-464c-b648-03ca34915aef]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.311838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e15eaaae-1020-4653-a1de-cae5b49bd2f9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.338038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[055d2271-474d-43f7-a7c3-ff2ab5920a4f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.375553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c4d8dad-4b5d-41d0-802f-a9fa612fa62e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 191745, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.398385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[476dab98-4602-455c-9fc1-cb0946558647]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.423446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b18f157c-523b-4513-a634-83bc0b90f2d5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.452267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63ad47fe-344f-4c10-9015-1c0fa9a9060e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.474566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2f6a3da-2f08-4c25-b8ba-403b0eda3bf6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.475132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:04.475437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:04.475749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:04.499720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0148be7-06f2-4706-852b-2c597cf1971d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 594063, 'reachable_time': 24632, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 191751, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.501128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:04.501413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:04.501851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:24:04.502119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:24:04.502456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:04.502719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:04.503019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '3079fda2-a673-4bf2-ba86-4126326cd8c8', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:9e:70:39', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '88bbc4fa-c162-4343-9551-ceded0fe88c3', 'device_owner': 'network:router_gateway', 'standard_attr_id': 4910, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.249', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:24:01Z', 'updated_at': '2022-07-01T10:24:03Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-3079fda2-a6), ns(qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:24:04.518068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-3079fda2-a6 not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:04.518347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ae77799-db04-4c23-aa81-4e0ca3cbbc78]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.522417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6ddabd1-e459-4aa7-9cbf-3d7ec2575d4f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.523409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-3079fda2-a6, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:04.523822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:24:04.524848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-3079fda2-a6, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:04.525524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-3079fda2-a6, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:04.525919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-3079fda2-a6, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:04.526308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-3079fda2-a6, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:04.526689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-3079fda2-a6, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '3079fda2-a673-4bf2-ba86-4126326cd8c8', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:9e:70:39'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:04.583483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82b8e450-11b3-477b-9e8b-f217ea7c78d4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.628885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b594ae48-b8d3-4056-b82f-57cefa2ea8c7]: Network interface qg-3079fda2-a6 not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:04.628885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:04.628885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:04.628885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:04.628885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:04.628885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:04.628885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:04.628885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:04.628885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:04.628885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:04.628885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:04.628885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:04.628885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3079fda2-a6 not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:04.630000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b594ae48-b8d3-4056-b82f-57cefa2ea8c7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-3079fda2-a6 not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.630437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:24:04.631029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:24:04.664495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d888ee88-50da-4fd4-ac2f-e8a6c3a59305]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.748056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8b401ac3-f4f1-4a17-9bc8-b359d9490c2a admin admin] Got routers updated notification :['88bbc4fa-c162-4343-9551-ceded0fe88c3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:24:04.844590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4d867ee-43ef-467d-9dbc-31f324ae6340]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:04.860211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e601018-1d8f-4288-82a7-c26c48d45cf4]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:05.058472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90419f95-190c-4c85-9e7e-fd0294b09051]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:05.058977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:24:05.059296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:24:05.059651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:24:05.059945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" released by "process_external" :: held 0.558s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-3079fda2-a6 not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3079fda2-a6 not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3079fda2-a6 not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:05.060731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3079fda2-a6 not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3079fda2-a6 not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:05.062866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:05.072567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b89d830-cd87-4025-af15-b2392ea15e05]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:05.073546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:05.074151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:05.074411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:05.074757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:05.078545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5069c7b-8aa5-4c31-8683-fe26a7091fde]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:05.081338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c97d4b4c-359b-4bdd-9af7-67cf84fb5cc6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:05.084092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c89d622f-f886-478d-82ac-9dcf518abbdc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:05.113397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71ce49c9-af01-422f-9ba9-5ce272e7f4c2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 594063, 'reachable_time': 24632, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 191782, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:05.124266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:05.125114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76d498c7-1ac5-4589-8c9d-4ccca94aae1a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3079fda2-a6 not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3079fda2-a6 not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:05.126388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:01Z", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:03Z", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id 8dbfe4a4-ded5-451b-8f14-5d67e11e7f3f. Wait time elapsed: 0.383 Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id 8dbfe4a4-ded5-451b-8f14-5d67e11e7f3f. Wait time elapsed: 0.384 Jul 01 10:24:05.129605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:07.625144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8bd3ee0a-4c1f-4db5-9c68-e1846e16fb50 admin admin] Got routers updated notification :['c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:24:07.625703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 1, update_id 8b73ff48-51f0-44bc-85e1-f2b260aad7d4. Wait time elapsed: 0.000 Jul 01 10:24:07.626007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 1, update_id 8b73ff48-51f0-44bc-85e1-f2b260aad7d4. Wait time elapsed: 0.000 Jul 01 10:24:07.626358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:08.892765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.266s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:08.893399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:08.894007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:08.894328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:08.894608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:08.899593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa255f89-101e-4719-a94d-e9563465cbfe]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:08.920752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[414fb489-eba7-46f2-8168-fb9dc67df31a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:08.941111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee79c447-2997-434b-a419-b2d98b5837c2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:08.969703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[345523ad-2c5f-42cb-a304-4a85158559ac]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 191826, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:08.990577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c85a38d-3568-4b5a-9044-2fce74a996fa]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.011823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06e55639-ab6b-433a-8d49-d05398056475]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.031661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8fa0544f-5baf-40d7-bd75-e7af814c7a48]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.052970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[284b6b3b-2e1e-4242-8837-75670e10da73]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.053557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:09.053837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:09.054132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:09.074617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02ba8d57-b414-4a9e-8f4f-c2e742a9102c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 594523, 'reachable_time': 40716, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 191832, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.075400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:09.075691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:09.076129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:24:09.076395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:24:09.076724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:09.076983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:09.077273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '84da0ba8-3462-4b8c-b194-99caf0f15496', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:6c:4f:08', 'admin_state_up': True, 'status': 'DOWN', 'device_id': 'c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'device_owner': 'network:router_gateway', 'standard_attr_id': 4924, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.135', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:24:06Z', 'updated_at': '2022-07-01T10:24:08Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-84da0ba8-34), ns(qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:24:09.091724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-84da0ba8-34 not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:09.092025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2bed0290-2977-40c8-8aca-0d8d46ddb4ed]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.095051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e2bd7a6-90ac-45e3-bd2d-388234ed09d7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.096043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-84da0ba8-34, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:09.096364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:24:09.097124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-84da0ba8-34, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:09.097790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-84da0ba8-34, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:09.098443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-84da0ba8-34, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:09.098623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-84da0ba8-34, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:09.099812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-84da0ba8-34, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '84da0ba8-3462-4b8c-b194-99caf0f15496', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:6c:4f:08'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:09.151444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91bf8e4e-d9bc-4a5c-83a3-e0077c481c7b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.182016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ddf9f42a-4a6f-47d9-9050-e9611a56b0fd]: Network interface qg-84da0ba8-34 not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:09.182016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:09.182016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:09.182016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:09.182016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:09.182016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:09.182016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:09.182016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:09.182016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:09.182016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:09.182016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:09.182016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:09.182016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-84da0ba8-34 not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:09.183101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddf9f42a-4a6f-47d9-9050-e9611a56b0fd]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-84da0ba8-34 not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.183101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:24:09.183470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:24:09.194369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3aa679e5-7b4f-4612-ba68-d779b914c719]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.398368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4cbc06b-981b-4715-8088-f9fd80ad651c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.435564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0900d5a-b264-411f-91c2-46276cc302fa]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.453414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.325s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:09.454080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:09.454812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:09.454930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:09.455201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:09.457472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db8f8822-bc5a-4a99-a6be-2b6ec2486c3d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.478464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3473a379-0da9-4946-8c86-9bdd9942f93a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.502992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb34986f-b29c-4b09-9a6c-48183c60c4dc]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.549711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0a6aad7-2938-4496-91f9-8059d2ec6cae]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 191853, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.572592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43f52ad0-3e00-4c5d-b1e0-03ab6f52fef0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.594744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9895219e-5bc0-4389-8840-f75b3b64dca4]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.618411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[845fdc5e-b70b-447f-a683-55efe1f9dd62]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.631853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad07b101-53e8-4902-8838-33ebda74a548]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.632270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:24:09.632580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:24:09.632955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:24:09.633355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2" released by "process_external" :: held 0.557s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-84da0ba8-34 not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-84da0ba8-34 not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-84da0ba8-34 not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:09.634212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-84da0ba8-34 not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-84da0ba8-34 not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:09.636591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:09.641120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a9ad090-57f8-4b3d-ac1c-df9ea8dbfdc7]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.641611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1fc9683-936c-4cc6-84e4-aaf48394cbd3]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.642708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:09.643024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:09.644294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:09.644836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:09.645557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:09.646016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:09.646750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:09.647251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:09.653480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[baf00b42-5983-409b-b5b3-0380783e38e4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.657185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db8639cc-de79-4a77-a5c9-6d3a7a832ed7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.658389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7d745f3-d07d-486a-8120-06d1fb41b016]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.680882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:09.683207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e2ad94b-8ce7-4c95-8d4f-40917c551008]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.686328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8b3d4c8-38ff-483f-bb34-0c6fab9a2d9c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.687339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb513368-cd63-4081-a166-2465128e4caa]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 594523, 'reachable_time': 40716, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 191860, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.688179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:09.688571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:24:09.695110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:09.695943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:09.696403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:09.696927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:09.697400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:09.698912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:09.699410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c4ec7ad-8b2b-4b37-af9b-ebb22e40c2ef]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-84da0ba8-34 not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-84da0ba8-34 not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:09.714174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-967313640-router", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135" Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:07Z", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6c:4f:08", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4924, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:08Z", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:24:09.728030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:09.767061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 8b73ff48-51f0-44bc-85e1-f2b260aad7d4. Wait time elapsed: 2.142 Jul 01 10:24:09.767815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 8b73ff48-51f0-44bc-85e1-f2b260aad7d4. Wait time elapsed: 2.142 Jul 01 10:24:09.768521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:09.780667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00aa7cc2-0579-43d2-92fb-fdcf1a8e92f9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.828841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c07f6de8-8d3b-4c0e-817c-a1804273a901]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:09.828841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:09.828841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:09.828841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:09.828841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:09.828841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:09.828841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:09.828841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:09.828841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:09.828841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:09.828841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:09.828841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:09.828841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:09.829791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c07f6de8-8d3b-4c0e-817c-a1804273a901]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:09.830651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:09.832225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:09.844017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c9264bd-89a9-4e0c-80c3-ee13e1226959]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.846488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:09.846824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:09.847084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:09.847534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:09.852804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[353af250-cf52-442c-9607-44777d47a9b9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.854182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[717dba3d-95d4-40e6-b445-b44f8c076500]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.855185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95b62165-c5f6-472e-b9f4-67d3fe59a77b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.894613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdc8bcaf-2197-4bc1-8e44-3c7b40deab4b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 594581, 'reachable_time': 15696, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 191871, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.897521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:09.898041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7accfe6-b6ed-412f-ad01-3d095d781211]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:09.900982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:07Z", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:09.905224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:09.926310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 4f3c4b9f-05b0-4205-ab6f-68576a2b0f73. Wait time elapsed: 6.945 Jul 01 10:24:09.926310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 4f3c4b9f-05b0-4205-ab6f-68576a2b0f73. Wait time elapsed: 6.945 Jul 01 10:24:09.926310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:10.592114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b1cad15b-8aec-4c36-8593-a721080d3b6a admin admin] Got routers updated notification :['c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:24:10.709259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.803s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:10.710284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:10.712028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:10.712476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:10.715590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:10.717660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3024f20d-3b88-4131-ac73-17bf028542e6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:10.750814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43b89745-b779-476f-9999-312e9570e207]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:10.777411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd065c10-9ac1-48fe-8419-1aff8c617477]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:10.886041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c0b634d-9563-4573-b8ed-703f9e95aacb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 191882, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:10.969815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c1d81af-f159-4a82-971b-384d9420f4bd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.033070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0b63c7f-534e-4fbe-97f4-512aef773847]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.092841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26b22993-c377-4a9a-8b21-b521882153d8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.128457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eba6da15-e2e6-4ce1-a1bb-679903fac219]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.129282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:11.129713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:11.130134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:11.130560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:11.159012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:11.159589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50591415-da4b-46c0-af81-ced8d68f34d2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.162565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d854bde8-0274-4178-8aa0-dac280d44500]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.163618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:11.237503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:11.238172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:11.253108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:11.253508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:11.253902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:11.311838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6aaa428-1ccd-4025-ac2d-8cbfe3ad02e0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.358566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0080c7db-52c4-4cfb-9d28-be1d33d05604]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:11.358566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:11.358566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:11.358566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:11.358566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:11.358566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:11.358566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:11.358566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:11.358566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:11.358566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:11.358566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:11.358566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:11.358566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:11.362266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0080c7db-52c4-4cfb-9d28-be1d33d05604]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:11.363539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:11.366766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:11.371470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c6fbeb2-901f-4aa1-8a42-4814c0dd3350]: (4, ['qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.372522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:11.373009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:11.373415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:11.373894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:11.375167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ad08760-06f6-4533-8cad-2c3f15f602d8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.386839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa683e0f-e0c7-4866-965b-838f463ec480]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.409099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1acfbb8c-d41a-4f1a-b011-8faf67fd2b95]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.457807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d085218a-6a7a-4c37-ac26-ed4d959e5f80]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 594714, 'reachable_time': 30920, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 191956, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.466203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:11.466455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd6b9078-3fe3-4aff-a883-4af06fc216a6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:11.467528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:07Z", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:11.471769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:11.486751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 8dbfe4a4-ded5-451b-8f14-5d67e11e7f3f. Wait time elapsed: 6.739 Jul 01 10:24:11.487488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 8dbfe4a4-ded5-451b-8f14-5d67e11e7f3f. Wait time elapsed: 6.739 Jul 01 10:24:11.487488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:15.062675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.575s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:15.063805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:15.065009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:15.065548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:15.066041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:15.067610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f15b6cbc-4535-4129-ac47-776a9bce4f54]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.085094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d39d63e-53eb-4d96-a851-7cf2e3530dac]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.122954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc95b4b8-2095-40d1-97da-e1543a425653]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.141168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:15.141745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:15.142278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:15.142709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:15.156897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c8b2a8f-bff0-4296-921c-e4addba6d0bc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 191984, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.182117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c645885-276a-48e9-b63e-736801e9948d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.214162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[309e760a-070d-4dee-b470-302ffc551655]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.246828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e02a5c18-3ab0-4532-89a8-137e42e05c3e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.264355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:24:15.271435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d26b5bf6-fbba-4e09-855a-f06d6a003819]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.272221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:15.272499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:15.272769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:15.273051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:15.296353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:15.296990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[498639a4-ec33-4b09-8726-bf18ddc47850]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.299851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e57aafa3-7343-4e1a-bd26-0d8af35f52c6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.301027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:15.343617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:15.344644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:15.345867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:15.346491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:15.347114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:15.431709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5215d188-bbe4-45f3-b422-a1407395329e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.466439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8309ae53-627e-42d9-9a98-b240e15d2ff8]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:15.466439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:15.466439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:15.466439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:15.466439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:15.466439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:15.466439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:15.466439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:15.466439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:15.466439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:15.466439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:15.466439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:15.466439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8309ae53-627e-42d9-9a98-b240e15d2ff8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:15.467722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:15.469823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:15.472523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4feb613d-40cd-4289-bad9-0e469a418f23]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.472523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:15.472523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:15.472523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:15.472523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:15.472523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e571068c-7489-489d-9769-0e28a54783b0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.473557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52b30ec0-b68a-4866-8597-d45e6bd5ae0c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.474781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1b68b2b-6f50-44a6-af8f-689e15793e56]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.505533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70c2e99d-ae9f-41f9-bf49-ebb01ca09b98]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 595141, 'reachable_time': 18804, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192002, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.521310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:15.521632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3acfba3-e60b-4458-b175-4fe98dd645c8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:15.522736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:11Z", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:15.527192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 4f3c4b9f-05b0-4205-ab6f-68576a2b0f73. Wait time elapsed: 12.563 Jul 01 10:24:15.538852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 4f3c4b9f-05b0-4205-ab6f-68576a2b0f73. Wait time elapsed: 12.564 Jul 01 10:24:15.538852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:16.482708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.957s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:16.483314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:16.484820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:16.485472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:16.485946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:16.487566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b535ee5b-f8cd-4cfa-b671-5409bdf8b81e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.519185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa1338a8-7c39-4291-acf8-bf482422118b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.544801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef5e957d-54f5-4bc1-95db-0a04138ac596]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.587614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70766fd9-ecb7-4f28-852f-532f2ecefaf4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192005, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.655131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae24676f-2209-4776-88e6-3d653f5f3f92]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.685215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15b4e6a8-72a6-4002-af63-99c0aa9ea46b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.710018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf9a0bef-d235-4cfd-a967-b05a3a4d8bd3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.734042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[999e2f59-e156-40f2-9901-54fcb12be82e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.734627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:16.734924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:16.735492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:16.735932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:16.760143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:16.760669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e71eb71-a481-4dd2-beec-8d19d28ab5db]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.763631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[366f581a-3dbc-42c0-ab6f-e80ed4b35762]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.764810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:16.806291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:16.807266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:16.807897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:16.808430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:16.809244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:16.882028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 7.114s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:16.882241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:16.884416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:16.884835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:16.885466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:16.886762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a65fcabe-3df0-4f84-8867-20d30eb7554d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.896210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85fe5e36-76ce-461f-bf0c-6c1073859260]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.925539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[724ebe05-dce1-4514-ad1e-d1407c005561]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.949695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c7ae76c6-b331-4f51-9ac0-1bb4f116a4d8]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:16.949695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:16.949695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:16.949695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:16.949695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:16.949695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:16.949695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:16.949695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:16.949695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:16.949695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:16.949695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:16.949695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:16.949695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:16.951084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7ae76c6-b331-4f51-9ac0-1bb4f116a4d8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:16.952326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:16.955057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:16.959301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea858853-3e8e-4022-b2a7-2dc3b0c107cf]: (4, ['qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.960355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:16.960768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:16.961772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:16.962215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:16.963708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16451567-f38e-4185-9217-e6338c60ba83]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.964959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5692b933-f1e2-4537-b84a-36e2ed72fa78]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.966429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1dc93b6f-1ebc-4685-999c-7e73b59a4975]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.978080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[377f8d75-3f4a-42e1-a278-1f939ea6240c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.993782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25fec896-11ed-4469-a4aa-8a067d04df13]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 595284, 'reachable_time': 28472, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192028, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d12064c-61bb-48f3-b471-f37a96125493]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:16.999756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:14Z", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 8dbfe4a4-ded5-451b-8f14-5d67e11e7f3f. Wait time elapsed: 12.255 Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 8dbfe4a4-ded5-451b-8f14-5d67e11e7f3f. Wait time elapsed: 12.255 Jul 01 10:24:17.004970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:17.021526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a458f15-e1ad-44a2-b583-e92ce9f09651]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192029, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.045506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[117d43ad-400f-4d99-b6e1-1275f492163d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.069235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48270978-e546-4ab6-ac4c-3a6d8bdb8556]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.087301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7978d4a5-a9e6-4c29-bbf3-e80e551fa0fa]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.112189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[114bee0a-05f6-418f-a9bb-eb959bb4655e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.113318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:17.113735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:17.114183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:17.114616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37b9f496-8b52-43c8-90fa-4e582241f062) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:17.141403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:17.141809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39cf88ad-c335-4b84-8394-1fe9a4abcd9c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.148322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd552bef-f700-41f4-8d8e-0d5d7100e512]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.150598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37b9f496-8b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:17.151037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:24:17.151943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37b9f496-8b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:17.152875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:17.153549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:17.154172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:17.154782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37b9f496-8b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37b9f496-8b52-43c8-90fa-4e582241f062', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:5c:e7'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:17.210575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1035da8-09ab-4cfa-9c6c-df1e94a14e73]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.269848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cadc8bd5-4bd9-4d64-a345-2ec34091b618]: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:17.269848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:17.269848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:17.269848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:17.269848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:17.269848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:17.269848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:17.269848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:17.269848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:17.269848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:17.269848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:17.269848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:17.269848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:17.270845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cadc8bd5-4bd9-4d64-a345-2ec34091b618]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:17.271038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:17.274752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:17.278467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e2f07d2-fbc3-4895-9000-582a37170f09]: (4, ['qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.280850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:17.281449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:17.282464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:17.282968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:17.286105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1ea7afd-6a3b-4f5f-a323-07d121d5c9eb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.287349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7764aad-161b-4721-8285-52c553717bac]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.295068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[125f5477-6e43-4dab-ad12-2f3bcf1314d1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.332999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[133408c0-8088-45fd-83b0-a3f56591eefc]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 595328, 'reachable_time': 41928, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192046, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.335739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:17.337239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea200a18-6ef3-4c44-9fde-3b640662f700]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:17.338508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-967313640-router", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135" Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:09Z", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6c:4f:08", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4924, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:13Z", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37b9f496-8b52-43c8-90fa-4e582241f062", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c369e92b-9b88-40b2-8c7d-00effae68044", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:5c:e7", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4929, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:08Z", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:14Z", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:17.342415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:17.359423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 1, update_id 46a7f3d0-fe2b-4d54-bf16-e1d513957999. Wait time elapsed: 6.765 Jul 01 10:24:17.359423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 1, update_id 46a7f3d0-fe2b-4d54-bf16-e1d513957999. Wait time elapsed: 6.765 Jul 01 10:24:17.359423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:18.303642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.945s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:18.304403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:18.305509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:18.305916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:18.306264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:18.307700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a69d3854-ef21-46e7-8784-dab327528e02]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.339913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25918e72-4eab-4eb6-adbd-d6c44d63b6c8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.368461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9df91911-c374-428c-8b34-a11b90416254]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.409458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e01f76f8-e2ad-4645-814d-dfc8dbff7519]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192049, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.455070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a7aff57-a467-4b3e-a7d6-2d0e6594a0da]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.479335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6985eb7-38b0-4353-bb08-4d7bfdb40363]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.522388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[947c4392-b7c2-40e3-85f5-47db28bfda6d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.552688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0120bdc6-0dd8-45c7-9dce-9377bce522cf]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.553694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:18.554222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:18.554716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:18.555301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37b9f496-8b52-43c8-90fa-4e582241f062) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:18.593649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:18.593931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92c5f498-9b8f-43cc-b56c-97aa1ea75f2a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.600518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d526b82-31de-44d6-9479-01e76458d3a3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.602484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37b9f496-8b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:18.647715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37b9f496-8b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:18.650161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:18.650884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:18.651440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:18.652296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37b9f496-8b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37b9f496-8b52-43c8-90fa-4e582241f062', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:5c:e7'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:18.715318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5b20c7f-4659-44e5-8103-87195e5fc621]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.778232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3ddc390e-47f7-4d05-b27f-24858f08f62a]: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:18.778232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:18.778232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:18.778232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:18.778232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:18.778232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:18.778232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:18.778232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:18.778232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:18.778232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:18.778232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:18.778232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:18.778232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:18.779128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ddc390e-47f7-4d05-b27f-24858f08f62a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:18.779453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:18.781247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:18.785787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f81f71d-f718-4773-abb5-2774955dce56]: (4, ['qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.786997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:18.787317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:18.788005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:18.788400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:18.790031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c7ec2e9-0dc7-4523-a312-6a850bdccdd6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.791289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7bc49f7-6aa0-454a-b977-58f55499636b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.793093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3ca0125-6209-4980-86d4-e644020eaeae]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.847186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66338eeb-b439-42aa-9155-4eaf7a7b8f66]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 595467, 'reachable_time': 44684, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192073, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.855959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:18.856419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fff9b311-80d0-4cad-9302-ddd433618182]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:18.859067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-967313640-router", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135" Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:09Z", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6c:4f:08", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4924, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:13Z", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37b9f496-8b52-43c8-90fa-4e582241f062", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c369e92b-9b88-40b2-8c7d-00effae68044", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:5c:e7", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4929, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:08Z", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:14Z", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:18.863149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:18.881188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 8b73ff48-51f0-44bc-85e1-f2b260aad7d4. Wait time elapsed: 11.244 Jul 01 10:24:18.881188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 8b73ff48-51f0-44bc-85e1-f2b260aad7d4. Wait time elapsed: 11.249 Jul 01 10:24:18.881188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:20.882890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.884s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:20.883789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:20.884210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:20.884570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:20.884814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:20.886066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60344ec6-a590-4d65-b7e8-026f818401d7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:20.910357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad86d750-ec0b-45db-88c6-bb1f4f39f5b8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:20.938169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59da2a9c-fbfd-4e6a-bcb7-925a76338388]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:20.980922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[935653fb-1f9b-4469-a7ca-1b5a4a5c51e5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192165, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.014246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[455b6b43-2113-4bfd-a854-e75f5d194b51]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.056290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b0338df-5daa-461d-a30d-7a7ffbd98722]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.084284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df7798ed-3571-4762-b3e4-fa7defcbc85b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.112468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d48df349-a6f8-4d43-bdb3-563dbc61fc48]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.113632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:21.114046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:21.114438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:21.114897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:21.135304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:21.135887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d7b334e-90d9-4282-8f79-60b84c5cd831]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.138975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eccb66e8-2b37-4768-9ac0-9f738efca0f4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.140189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:21.181925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:21.195336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:21.196574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:21.196977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:21.197367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:21.249300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0699830-c297-4171-851c-786a64e29f80]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.297073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d677f066-0e65-4ca9-8794-938cd60f6db8]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:21.297073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:21.297073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:21.297073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:21.297073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:21.297073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:21.297073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:21.297073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:21.297073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:21.297073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:21.297073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:21.297073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:21.297073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:21.298518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d677f066-0e65-4ca9-8794-938cd60f6db8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:21.299654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:21.302321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:21.307311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee99008b-b364-4612-b744-6764a842211a]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.308108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:21.308437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:21.308689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:21.309032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:21.310139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a328a9cf-9087-46d8-a055-fe53af79f23d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.311490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ac5b521-0879-4889-9729-db85b0ccd689]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.312300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8b4f302-7139-4d6a-8851-7731456e9c72]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.333493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb742fa4-3fe0-4690-b8af-6d00f28d5359]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 595724, 'reachable_time': 22304, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192185, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.339342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:21.339683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8e934aa-6a56-4e19-8830-1cab50eac8a4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:21.340685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:17Z", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:21.343266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 4f3c4b9f-05b0-4205-ab6f-68576a2b0f73. Wait time elapsed: 18.381 Jul 01 10:24:21.365578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 4f3c4b9f-05b0-4205-ab6f-68576a2b0f73. Wait time elapsed: 18.382 Jul 01 10:24:21.365578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:22.293205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.951s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:22.294307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:22.296053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:22.296562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:22.297022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:22.298416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9070832-c281-41c9-b8bd-7ca7da40f147]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.324249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20f6b116-6664-47a7-8434-22546b53c932]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.346273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b0e479c-1e37-4f45-9c2a-c854c3ff3805]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.378737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0af459ea-a66d-4d28-ad10-cc7f2d2dee2d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192188, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.401527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1054d0b-f8b8-48a2-bd6e-deb89bcf5b28]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.422737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8af9e587-872c-4481-a437-80b3b2dff5a9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.448042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63396b88-c021-409f-a8ac-b5511b8a2d06]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.476712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[201990cb-2288-40d0-8bc0-3d7faf84db8a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.477658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:22.478132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:22.478470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:22.478819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:22.503970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:22.504145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e369150-c052-4d42-a5e3-5e2e8232a48e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.507099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2888c2c-aaed-4f68-8c68-ad0c0b07b281]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.508806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:22.548925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:22.550129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:22.550129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:22.550358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:22.550966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:22.605606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ca259e2-5b23-43af-b2c2-2084b30b05ed]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.653520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[33cf7d8d-5794-449a-9ff5-53d559554a11]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:22.653520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:22.653520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:22.653520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:22.653520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:22.653520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:22.653520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:22.653520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:22.653520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:22.653520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:22.653520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:22.653520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:22.653520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:22.654598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33cf7d8d-5794-449a-9ff5-53d559554a11]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:22.654700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:22.656784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:22.659268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de86fcc6-fde5-429f-8357-67d5a61b05ee]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.659268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:22.659604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:22.660422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:22.660422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:22.661892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc4d7fc3-2052-4faf-9bc3-788d170ed8bb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.663422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8b853bc-72f3-482b-bced-ac30be31b339]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.665390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[690f5b8d-e607-443b-a92c-8116ce15b8d3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.699880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c4613dd-2988-4b5d-b3a2-dd4139bffb93]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 595864, 'reachable_time': 24412, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192207, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.703522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:22.703844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8a66e03-5efb-4bdb-a7d9-c7b65649ec54]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:22.704964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:21Z", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:22.707942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 8dbfe4a4-ded5-451b-8f14-5d67e11e7f3f. Wait time elapsed: 17.963 Jul 01 10:24:22.722881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 8dbfe4a4-ded5-451b-8f14-5d67e11e7f3f. Wait time elapsed: 17.963 Jul 01 10:24:22.722881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:23.075473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.199s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:23.075744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:23.076715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:23.077190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:23.077605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:23.079249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4966c7d-a256-4309-aef8-25eca7a6662d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.093467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b77968ef-9244-4014-a02b-7ca424e6a27c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.116799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c97ab92-3d21-4932-86fb-adaf211d9201]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.147062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05b5fcff-68b7-478f-a202-af3524f96d7c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192210, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.169259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[040bc5e9-0a95-43a8-a029-fdf1ffba5a74]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.190076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e037f936-24eb-4964-a286-a98f2417d044]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.212776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a86d9c0-2dfb-476c-8b9b-2149737c8884]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.246739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f9caeca-2549-41b6-a4b7-4df8debc7d8d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.247436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:23.247790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:23.248825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:23.249169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37b9f496-8b52-43c8-90fa-4e582241f062) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:23.271735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:23.273300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b70d444-9703-4054-bee5-df1574d0d293]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.277069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6bd1b6a-2e56-4ca5-a3ea-2833cc95a57c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.278133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37b9f496-8b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:23.316640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37b9f496-8b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:23.317890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:23.318684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:23.319507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:23.320037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37b9f496-8b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37b9f496-8b52-43c8-90fa-4e582241f062', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:5c:e7'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:23.378359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bbc1730-6bfb-455f-987a-a8bec9074dfe]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.415433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0ce5faf1-4cda-4f60-a353-d2825a917a02]: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:23.415433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:23.415433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:23.415433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:23.415433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:23.415433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:23.415433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:23.415433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:23.415433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:23.415433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:23.415433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:23.415433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:23.415433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:23.416994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ce5faf1-4cda-4f60-a353-d2825a917a02]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:23.419181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:23.422301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:23.428251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[757429f2-5034-429b-aa4c-68afe151b834]: (4, ['qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.429304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:23.430686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:23.432040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:23.432546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:23.437471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df4e6d77-ecd6-4768-84b3-3a17d51d7401]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.438419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1130bb6-0f6e-49bd-aa00-2f0e60447f0d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.439404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[355496c2-c7cb-45f0-8746-4a7cbd01a4e7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.463855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c6233b5-3365-454f-9ebd-884785662b16]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 595940, 'reachable_time': 35980, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192228, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b21b78e-aa77-4e41-a59e-e4f4c0ad2b2c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-967313640-router", Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135" Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:23.468836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:09Z", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6c:4f:08", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4924, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:13Z", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37b9f496-8b52-43c8-90fa-4e582241f062", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c369e92b-9b88-40b2-8c7d-00effae68044", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:5c:e7", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4929, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:08Z", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:14Z", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 46a7f3d0-fe2b-4d54-bf16-e1d513957999. Wait time elapsed: 12.876 Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 46a7f3d0-fe2b-4d54-bf16-e1d513957999. Wait time elapsed: 12.876 Jul 01 10:24:23.472846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:24.271019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.802s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:24.271379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:24.273077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:24.273300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:24.273781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:24.275670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf8d613f-ad2e-41d0-a3d1-4f4a711a2dba]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.292304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[423b6b79-e4c9-4679-97b3-3f7cbf983cd2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.313466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd996ffc-003a-40e1-8f80-fb31ccc51109]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.357181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53864582-4149-476e-95b5-b5db17def9bf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192232, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.388978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[afa51b5e-f045-440a-82b7-c7c5b34d46be]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.412956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b3d8fcc-d9a7-46d8-b2ad-81ac776b55f2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.449088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a93078f0-ef77-4e35-a416-33912e41ec13]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.470070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e491ef3d-7314-47ed-9342-3d5add0f0081]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.470731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:24.471074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:24.471440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:24.471803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37b9f496-8b52-43c8-90fa-4e582241f062) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:24.490795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:24.491413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f20d32e7-d85c-4f1c-bc4a-d8fd14982dd1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.494585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33c7248e-ea75-48a6-bf6d-097ce24a0c13]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.495812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37b9f496-8b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:24.528856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37b9f496-8b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:24.529551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:24.529943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:24.530326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:24.530707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37b9f496-8b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37b9f496-8b52-43c8-90fa-4e582241f062', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:5c:e7'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:24.589749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46fe7d1b-e3b5-4fa7-8193-071fcab8241e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.633592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bd358ef4-2d8f-4077-ad5c-9184a73700e3]: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:24.633592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:24.633592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:24.633592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:24.633592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:24.633592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:24.633592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:24.633592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:24.633592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:24.633592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:24.633592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:24.633592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:24.633592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:24.635094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd358ef4-2d8f-4077-ad5c-9184a73700e3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:24.637226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:24.638854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:24.640906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[979a14c7-89d7-44b5-b9e4-4c31b8602759]: (4, ['qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.640906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:24.640906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:24.640906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:24.640906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:24.641949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3e82fee-641c-4d23-a7a9-78fc5f31b597]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.644115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63cf0556-821e-4e5a-a6e7-ed55b0e5bd98]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.645420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5214534c-8851-49b8-936f-75d8954cd7bb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.667874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b5c5f85-76dd-4a91-a802-62e81ad434d5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 596062, 'reachable_time': 25576, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192250, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.670172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:24.670472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db631b86-209b-4259-9f5b-13cd4df3afb0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:24.671782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-967313640-router", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135" Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:09Z", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6c:4f:08", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4924, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:13Z", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37b9f496-8b52-43c8-90fa-4e582241f062", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c369e92b-9b88-40b2-8c7d-00effae68044", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:5c:e7", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4929, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:08Z", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:14Z", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:24.674553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 8b73ff48-51f0-44bc-85e1-f2b260aad7d4. Wait time elapsed: 17.048 Jul 01 10:24:24.685698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 8b73ff48-51f0-44bc-85e1-f2b260aad7d4. Wait time elapsed: 17.048 Jul 01 10:24:24.685698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:25.524808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.817s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:25.526251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:25.528043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:25.528770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:25.530095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:25.532344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1c7d069-a0b9-481a-9fc2-7677f6417e39]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:25.559505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a2f3022-4f48-474b-8992-fef987db7e74]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:25.594404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00010218-8d6e-44d5-9e5f-d9f56ec32ac1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:25.638781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4e8735f-94f5-449c-bfd5-12e9633ece18]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192273, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:25.682936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4766aff2-9c3c-4f6b-9fa1-b962f2fa5346]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:25.728981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6dfd28c-7a90-4786-aeee-3f39534c7b38]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:25.771624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4d32e12-bab2-46f2-883d-66e911f188ac]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:25.811732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a092d551-3bde-4368-9506-7d7efac3259f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:25.812546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:25.812970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:25.813384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:25.813808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:25.842998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:25.843572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d963317-3d23-4dc2-b281-1f88b7be0e4b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:25.846656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[229192ca-ecbb-4521-b08b-64523ea96b15]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:25.847826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:25.882066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:25.883158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:25.884134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:25.884679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:25.885213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:25.966012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0065fe8c-c67b-4ba7-97d4-0a88a0f2e7c1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:26.016763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b0f2a478-ccc7-46d9-a57f-11bb7248462a]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:26.016763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:26.016763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:26.016763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:26.016763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:26.016763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:26.016763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:26.016763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:26.016763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:26.016763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:26.016763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:26.016763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:26.016763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0f2a478-ccc7-46d9-a57f-11bb7248462a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:26.018265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:26.020283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:26.025298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73584c14-6fcf-42f5-bf33-eb03bf6bd19c]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:26.025298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:26.025298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:26.025298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:26.025298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:26.027463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fe85299-89a6-4316-b135-e9625abd7c68]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:26.028858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4303390a-480f-4293-84e6-1c99917d4aa0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:26.030070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57b508aa-4395-4717-b322-84e9277476ea]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:26.063728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[997eb266-b18c-4102-b0da-838fb7e84865]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 596190, 'reachable_time': 21940, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192292, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:26.076574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:26.076839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04c50d36-7eb0-4c82-ae2c-1370e16bf249]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:26.078270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:23Z", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:26.080930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 4f3c4b9f-05b0-4205-ab6f-68576a2b0f73. Wait time elapsed: 23.119 Jul 01 10:24:26.093578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 4f3c4b9f-05b0-4205-ab6f-68576a2b0f73. Wait time elapsed: 23.120 Jul 01 10:24:26.093578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:27.410675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.330s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:27.411640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:27.412756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:27.413452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:27.413892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:27.422675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[196b5029-c0e2-476f-aed2-afd909bc7c26]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:27.439221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1045892-638b-4b85-b961-398f63a854a4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:27.528425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[763f6e0c-3e71-425c-bbc1-dc8231cefe33]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:27.588516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e385a162-22ff-484e-9d2c-bc00d41c06cc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192297, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:27.624470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d406eb9f-3b48-47ec-a04c-77d5bc455fd0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:27.670730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[818049fb-5706-4519-a427-847173cfcf2f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:27.734953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1dcac59f-a881-4779-94a1-6042944ba37c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:27.781081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46a1326e-4a5c-463a-8705-7ba87e8b2edc]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:27.781870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:27.782306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:27.782724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:27.783157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:27.825294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:27.826149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cb775c4-7e71-4331-91c1-afd82201bb03]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:27.829385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fbb9d13-1e63-4f86-a154-b5222c4edbb0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:27.830565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:27.871859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:27.873982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:27.874664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:27.875236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:27.875801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:27.945305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2367d955-a651-4c1b-b5c2-a28d6704b132]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:28.008685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2a441bb4-20de-4394-805b-7645d1c71d29]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:28.008685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:28.008685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:28.008685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:28.008685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:28.008685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:28.008685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:28.008685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:28.008685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:28.008685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:28.008685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:28.008685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:28.008685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:28.010400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a441bb4-20de-4394-805b-7645d1c71d29]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:28.011504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:28.014217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:28.018232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60c5eb26-de42-451a-ac3c-a6d9fd9dcda1]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:28.019241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:28.019793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:28.020189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:28.020676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:28.021858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f976d258-fc5b-44f2-9845-c9ee026217d6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:28.022997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdff2bf0-1b1c-42f5-8c18-0026ceadf4ad]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:28.025340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bdfa9fe-31f3-45e7-a48c-891181f23534]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:28.063382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47084d88-66f6-4a9d-9d55-b15b749859c1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 596385, 'reachable_time': 31848, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192316, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:28.066055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:28.066509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8412d5a9-c3ae-4c12-8a58-810daa2a456a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:28.067708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:23Z", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:28.073223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:28.085385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 8dbfe4a4-ded5-451b-8f14-5d67e11e7f3f. Wait time elapsed: 23.332 Jul 01 10:24:28.085385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 8dbfe4a4-ded5-451b-8f14-5d67e11e7f3f. Wait time elapsed: 23.332 Jul 01 10:24:28.085385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:29.662200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.988s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:29.663570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:29.665188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:29.666609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:29.667164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:29.669024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55db30c0-9811-40f2-8712-d626b856cda7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:29.685299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8f408d4-4eb3-445b-a2c4-dbebe217501d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:29.706106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07befc79-e472-4db1-93f3-19e43116a7c8]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:29.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a0697df-982f-49d2-85db-4b76dc8bbd05]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192379, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:29.770771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0daad548-351b-4165-9985-175b491c426e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:29.799808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d156ce47-664c-4f67-ad5f-244e3da21136]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:29.837266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d87979b-03ed-4ed3-ba3a-363afeb73ed6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:29.892013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b49fb10e-1b43-4a40-8e0b-673fe2467994]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:29.892954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:29.893594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:29.894102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:29.894556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37b9f496-8b52-43c8-90fa-4e582241f062) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:29.924920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:29.925492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1eaa02b0-e06b-453c-b984-042f5fbab821]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:29.929177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca69a5ff-866d-43f1-a05f-7921e36f8886]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:29.931225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37b9f496-8b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:29.972955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37b9f496-8b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:29.973918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:29.974486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:29.975035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:29.975615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37b9f496-8b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37b9f496-8b52-43c8-90fa-4e582241f062', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:5c:e7'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:30.024678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2c5f032-4d46-40ca-8931-527bb5f41193]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:30.091199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[accd9b94-6f6c-4a38-9f50-b8beb70cbe67]: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:30.091199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:30.091199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:30.091199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:30.091199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:30.091199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:30.091199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:30.091199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:30.091199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:30.091199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:30.091199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:30.091199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:30.091199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:30.093141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[accd9b94-6f6c-4a38-9f50-b8beb70cbe67]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:30.094724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:30.098259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:30.105102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3a2f4e2-0721-4967-9667-384d11f8f1f4]: (4, ['qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:30.106493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:30.107123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:30.107586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:30.108226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:30.109495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ede21054-fa10-4a48-934b-f270ae8c4a33]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:30.111301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1085ede-4e99-460a-89af-6454c243a583]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:30.112758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08d22066-ff54-4c56-9cfb-52f5e87a3e88]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:30.144133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04be30b5-c28b-4953-b1e0-d004d030e544]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 596600, 'reachable_time': 30868, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192397, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:30.152978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:30.153479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2334899a-14f0-4d27-ad5a-e568d480bac6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:30.154604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-967313640-router", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135" Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:09Z", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6c:4f:08", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4924, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:13Z", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37b9f496-8b52-43c8-90fa-4e582241f062", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c369e92b-9b88-40b2-8c7d-00effae68044", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:5c:e7", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4929, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:08Z", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:25Z", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:30.158743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:30.175790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 46a7f3d0-fe2b-4d54-bf16-e1d513957999. Wait time elapsed: 19.582 Jul 01 10:24:30.175790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 46a7f3d0-fe2b-4d54-bf16-e1d513957999. Wait time elapsed: 19.583 Jul 01 10:24:30.175790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:31.210582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.035s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:31.211265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:31.212104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:31.212432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:31.212715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:31.213862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06fe3e5e-d698-456b-9764-c98cd6801f70]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.237375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a17c7889-266a-43b3-bf18-e938aa6cb925]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.273118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd287662-312c-497f-b4a9-c70fc572099c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.311919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b1d3b0c-f77d-4bed-920b-34bb12dffe3f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192400, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.344478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[425a7706-2100-40f9-aac8-7c4a1f9dac9a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.374563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[912e7a2b-285d-447c-bfe9-6b1577f74d25]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.415559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e55b325-041b-4bbb-bbd2-5cbd2ed37118]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.462121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66d0aa46-3ee6-4af2-aada-750556616c4e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.465018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:31.466194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:31.466697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:31.467233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37b9f496-8b52-43c8-90fa-4e582241f062) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:31.492575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:31.493334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75de7555-1dab-4628-be07-ae72f4042441]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.496446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8412069d-beee-4698-9102-d4f5403034b4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.500184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37b9f496-8b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:31.539402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37b9f496-8b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:31.542039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:31.550283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:31.550971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:31.552145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37b9f496-8b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37b9f496-8b52-43c8-90fa-4e582241f062', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:5c:e7'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:31.598663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc77b4fe-1842-4126-ba40-e688b897f2b2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8fc82ea2-d390-4f4e-8ac9-34d2231da373]: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:31.656086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8fc82ea2-d390-4f4e-8ac9-34d2231da373]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:31.658343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bdb38c4-b728-4d31-9a09-3bc38715caab]: (4, ['qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27b742c7-67ac-4e5b-a358-58af02890ff5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.664705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0868e88-4c60-4ee3-b0ba-52f489541427]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.667571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb8b1207-bb5c-49f8-a901-0524ba95435a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.705924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c33f1408-f50c-4d5c-a76f-a0c0052b4e2a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 596757, 'reachable_time': 20820, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192421, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.713611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:31.713869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee110f2e-fa43-405e-8a35-2c3739f34b94]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:31.714915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-967313640-router", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135" Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:09Z", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6c:4f:08", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4924, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:13Z", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37b9f496-8b52-43c8-90fa-4e582241f062", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c369e92b-9b88-40b2-8c7d-00effae68044", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:5c:e7", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4929, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:08Z", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:29Z", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:31.717665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 8b73ff48-51f0-44bc-85e1-f2b260aad7d4. Wait time elapsed: 24.091 Jul 01 10:24:31.739203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 8b73ff48-51f0-44bc-85e1-f2b260aad7d4. Wait time elapsed: 24.091 Jul 01 10:24:31.739203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:32.843551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.767s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:32.844641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:32.846398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:32.846725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:32.847014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:32.849993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c773aa3-3876-4d4b-9231-7ff655f996f7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:32.864030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[975dc113-0699-45c3-a4cb-c4c03d5429e6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:32.886562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f55574b2-5161-48b3-9611-ca853544aeca]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:32.917403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ee49b22-1576-4755-aa7c-f9a6ac41dcb1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192424, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:32.955299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1ffee01-ee19-4291-8de5-557e8c61522b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:32.990284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89ce1187-0bf4-44c7-86b5-18e5d21752d7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:33.027032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0a2b64f-23ec-4a24-9fb2-87a9e53e2ee7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:33.067135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e73e6592-26aa-48eb-8a31-1760af72f350]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:33.067909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:33.068385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:33.068718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:33.069103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:33.101854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:33.102871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19064b85-f6e5-459a-9b22-afd8ad107640]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:33.112063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[685a8619-bdcc-4c23-9dda-fbfdfaf6e345]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:33.113434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:33.153837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:33.154799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:33.155559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:33.156099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:33.156661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:33.227928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61bf9ae4-d97a-4b79-8740-83e0accb4a2e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:33.294612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[83663696-cb47-41d9-b569-f20c552d6b08]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:33.294612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:33.294612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:33.294612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:33.294612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:33.294612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:33.294612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:33.294612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:33.294612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:33.294612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:33.294612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:33.294612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:33.294612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:33.296130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83663696-cb47-41d9-b569-f20c552d6b08]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:33.297238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:33.300099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:33.305422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[825c6ced-a266-480a-9e09-95609b76d977]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:33.308024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:33.308536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:33.308944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:33.309564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:33.313547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4368f297-0fa0-4055-bc65-34daa1f7f356]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:33.314757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c8a8e40-31e3-438b-9d9f-23648fa723f6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:33.316051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2aacc837-9543-4b10-b4db-c01faa2bfa12]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:33.337933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d3163f0-61c1-42c4-8c16-9599d7d48604]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 596918, 'reachable_time': 34508, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192444, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:33.347565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:33.348007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c84be09-9e51-40ce-8ab0-510b5b4e4b45]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:33.349226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:27Z", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:33.353698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:33.368180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 4f3c4b9f-05b0-4205-ab6f-68576a2b0f73. Wait time elapsed: 30.404 Jul 01 10:24:33.368180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 4f3c4b9f-05b0-4205-ab6f-68576a2b0f73. Wait time elapsed: 30.405 Jul 01 10:24:33.368180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:34.197494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.832s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:34.198132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:34.198986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:34.199448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:34.199710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:34.200885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[868f96fc-07ea-40c0-bb39-335014df2e9a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.213120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ca4ae1e-b3f8-4532-9748-f73822f96bca]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.235966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[356a94fa-54a6-4374-816b-e60c0465bfa7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.263985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd889d7d-a7ba-4855-bf0e-6fe307655eae]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192447, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.284447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[007af4ad-7306-4329-a040-5ee75da35503]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.325558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05896b86-af1c-4b6d-8951-c40dc6777001]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.351089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80e66779-ff96-4aa8-bf45-80c59fe5ee78]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.370830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2c551cd-c451-4aed-92b1-27cad2ec7bc4]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.371720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:34.372001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:34.372262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:34.372544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:34.388104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:34.389332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc132238-f8b6-4398-83c6-b02aa2d31321]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.392205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce5a7306-d2d3-4e13-9c52-47757343a693]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.393221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:34.433589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:34.437032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:34.438786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:34.439569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:34.440111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:34.486022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0ee17a3-5176-447d-babf-e6063d5ce2d7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.533507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1cecbb87-cb11-4ae8-8b23-3ac0a627f19d]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:34.533507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:34.533507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:34.533507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:34.533507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:34.533507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:34.533507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:34.533507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:34.533507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:34.533507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:34.533507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:34.533507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:34.533507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:34.535523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1cecbb87-cb11-4ae8-8b23-3ac0a627f19d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:34.536611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:34.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:34.540981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[889e10ae-1655-49dc-875e-17f2658df12a]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.540981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:34.540981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:34.540981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:34.540981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:34.542799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[385239f1-2c5f-407c-a9d5-5923d4a3668f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.544001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d7b844c-808e-45c5-97a8-569575053b8c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.544951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b7262e9-ddad-4a93-ae18-496eb43ff59f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.565223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[065d1490-8059-48ce-b702-a9d1e7f2915a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 597052, 'reachable_time': 17476, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192466, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.571811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:34.572316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe4fdbaa-b95b-4390-acb5-a4c313fc99ee]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:34.573635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:31Z", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:34.576844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3 Jul 01 10:24:34.588596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 8dbfe4a4-ded5-451b-8f14-5d67e11e7f3f. Wait time elapsed: 29.831 Jul 01 10:24:34.588596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 8dbfe4a4-ded5-451b-8f14-5d67e11e7f3f. Wait time elapsed: 29.831 Jul 01 10:24:34.588596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:34.749154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.033s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:34.749506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:34.750427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:34.750751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:34.751031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:34.752582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[130346f0-f302-442a-a378-d7418e63bbf7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.770741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9efbaabd-7839-4b55-9a05-833154de0e79]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.792099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2ac5b72-23bd-48a5-96f1-4e5790d2bc44]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.820904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[590d3b7b-c708-4fb3-baa3-f449bb256f41]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192469, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.841394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc750cab-3a9d-486e-af61-bf83f250569c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.861204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5811191-9006-4adc-8522-ade227dd8582]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.882702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4871db21-8172-4648-844a-1a2a9298caae]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.904082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43dad5d9-d424-4b84-baf4-228ca6e8a8b9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.905212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:34.905551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:34.905910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:34.906278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37b9f496-8b52-43c8-90fa-4e582241f062) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:34.920721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:34.921108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ef337af-4ee7-4c14-b2fc-bd92076d7009]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.923869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7cc9885-6731-43a6-8480-5c9b585dd4f4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:34.924832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37b9f496-8b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:34.955004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37b9f496-8b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:34.955712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:34.956132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:34.956516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:34.956898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37b9f496-8b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37b9f496-8b52-43c8-90fa-4e582241f062', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:5c:e7'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:35.016867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84e6b1e9-bf9a-4382-8290-58e24e04e933]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.051872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ba644618-8588-4a4c-9457-03f96a9564d2]: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:35.051872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:35.051872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:35.051872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:35.051872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:35.051872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:35.051872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:35.051872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:35.051872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:35.051872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:35.051872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:35.051872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:35.051872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:35.052978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba644618-8588-4a4c-9457-03f96a9564d2]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:35.053818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:35.057023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:35.059615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[959a1db4-be98-4ce6-92ec-16a2acf565cb]: (4, ['qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.059910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:35.060285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:35.060543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:35.060962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:35.062209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24f9478d-eb33-4561-86d4-44288ff00015]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.063132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39bc106a-6a2d-4f7b-9ab6-b82e316989a2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.064075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c70d650c-7929-465d-88fa-eb282369271d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.083885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01cf189f-d22d-4f4f-b0f0-ecdd387ff459]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 597108, 'reachable_time': 42704, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192487, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.085972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:35.086907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09f555c3-8123-4748-84b1-312e2f40b003]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:35.088089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-967313640-router", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135" Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:09Z", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6c:4f:08", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4924, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:13Z", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37b9f496-8b52-43c8-90fa-4e582241f062", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c369e92b-9b88-40b2-8c7d-00effae68044", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:5c:e7", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4929, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:08Z", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:29Z", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:35.091802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:35.102371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 46a7f3d0-fe2b-4d54-bf16-e1d513957999. Wait time elapsed: 24.499 Jul 01 10:24:35.102371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 46a7f3d0-fe2b-4d54-bf16-e1d513957999. Wait time elapsed: 24.500 Jul 01 10:24:35.102371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:35.798637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.706s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:35.799257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:35.799974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:35.800355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:35.800695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:35.802477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3cc9232a-600e-4b33-b981-8a70cf7224c6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.817260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[418f9761-358f-4755-9439-ccbd8216e3df]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.839763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6bf85114-3fb4-4fe1-9585-51444e27ce99]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.868795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07d060da-da22-4ed7-a741-2e0e3381a538]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192490, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.889149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5de99da9-a835-4228-857c-843369b6e10e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.909132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6ca9382-338b-4be7-97c5-9dfbbbd6361b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.931616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b84c8462-8c70-4620-8874-32774ba39b3a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.955261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62e1bbbe-54e4-493b-8e3b-9edcffd45f1a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.955873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:35.956146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:35.956411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:35.956709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37b9f496-8b52-43c8-90fa-4e582241f062) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:35.971605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:35.971907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d9b4144-ead3-4371-b653-ae9044d341fa]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.974408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1078ba07-272b-4585-8981-28d1fa164c2f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:35.975400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37b9f496-8b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:36.015906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37b9f496-8b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:36.016659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:36.017125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:36.017576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:36.018097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37b9f496-8b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37b9f496-8b52-43c8-90fa-4e582241f062', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:5c:e7'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:36.078468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a24c8d46-4143-48e1-94a6-56f14399365c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:36.117318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[086f3547-fe2b-4090-979a-65ef3eafcccb]: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:36.117318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:36.117318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:36.117318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:36.117318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:36.117318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:36.117318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:36.117318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:36.117318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:36.117318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:36.117318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:36.117318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:36.117318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[086f3547-fe2b-4090-979a-65ef3eafcccb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:36.118613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:36.120653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:36.122814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad4a582c-aec5-4fb5-88f1-02136541be24]: (4, ['qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:36.122814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:36.122814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:36.123108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:36.123354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:36.124439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecf381fe-2612-4190-a63a-65671bef761d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:36.125491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[252a00fb-2085-4e5d-828e-3ce13fefa377]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:36.126486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89dcc0f2-94a2-475b-957a-55131228fc19]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:36.151563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19f7d8a6-196c-4af3-ab15-6f4cba0aa7b1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 597213, 'reachable_time': 18472, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192509, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5170e457-0e4f-4528-aa49-641c665e3c70]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-967313640-router", Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135" Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:36.156817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:09Z", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6c:4f:08", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4924, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:13Z", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37b9f496-8b52-43c8-90fa-4e582241f062", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c369e92b-9b88-40b2-8c7d-00effae68044", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:5c:e7", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4929, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:08Z", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:34Z", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 8b73ff48-51f0-44bc-85e1-f2b260aad7d4. Wait time elapsed: 28.531 Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 8b73ff48-51f0-44bc-85e1-f2b260aad7d4. Wait time elapsed: 28.531 Jul 01 10:24:36.161569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:38.297917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.722s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:38.298500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:38.299299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:38.301678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:38.302061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:38.303737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6aabe5db-7b25-46bc-9e01-caca1f5db2f4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.317607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[744482bb-ac18-406e-a465-6a5d9f1a1f1e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.339985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4023c26-f18a-4cc6-afc5-cc970b6b4309]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.375239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63f91554-1266-4589-bee5-7dfefc13c8de]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192576, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.397762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ec07f6b-2b7d-41f2-b2c2-bd2367d31888]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.418754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d732879c-df18-4620-a141-bb89397af4da]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.442714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e2b8517-ce04-434e-a502-1ddeb6de3598]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.464645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c3b4081-898f-4c98-85e9-b96eba7fb05e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.465295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:38.465629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:38.465957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:38.466296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:38.482625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:38.482925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11b52653-d62d-4b66-a950-bbe73f600cd3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.485713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f2a199b-691e-4ca6-9269-ac5275eba598]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.486861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:38.516155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:38.516842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:38.517281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:38.517660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:38.518069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:38.568960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d338218-0c45-46a6-8cfe-6a2ea6c670c3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.600122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1c935dd0-5ed7-4766-8f1f-e26663eac36d]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:38.600122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:38.600122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:38.600122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:38.600122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:38.600122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:38.600122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:38.600122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:38.600122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:38.600122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:38.600122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:38.600122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:38.600122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:38.601244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c935dd0-5ed7-4766-8f1f-e26663eac36d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:38.601378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:38.604181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:38.607037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1951b6ac-83e8-47a0-a21d-62ca33afc4e2]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.607037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:38.607037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:38.607037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:38.607037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:38.607869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[186c163f-8157-4ede-b680-0c47319978fa]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.608867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80903e07-853f-4139-9076-4971b7b7872f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.609777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43179967-12f0-4b85-805e-c4970ceccc01]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.629146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[849a44fc-c1b0-4023-aa93-b0c1c09ba5a8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 597464, 'reachable_time': 39332, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192594, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.635686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:38.635686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00d28743-186e-4d5b-830e-d9455407d561]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:24:38.636182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:35Z", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 13, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:38.639435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3 Jul 01 10:24:39.850107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.691s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:39.850670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:39.854865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:39.854865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:39.854865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:39.856230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1826e31e-1f16-4762-b528-f12a65644298]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:39.886544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f4ab3d3-ff04-4551-8d52-a60f1eaf1653]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:39.908014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31fa515f-cb1c-473f-92a8-43d2e7faf53d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:39.943835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1402739-b12b-47e1-8c3c-8f8bd154db8f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192598, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:39.974661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5057e1e-8b7d-4ef2-871b-5e08f06edd21]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:39.996593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61b1212f-a3a5-4766-addc-4f72d6d3e983]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:40.018068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c797c135-5df5-4035-a42d-ce2615a4d1ef]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:40.038890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5422077-398d-49a5-8476-26894c54e7f9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:40.040232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:40.040535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:40.040860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:40.041200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37b9f496-8b52-43c8-90fa-4e582241f062) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:40.055740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:40.056016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[782ea1c5-19cc-4e0c-9f6b-15523d89beca]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:40.058765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a31e722-d4fa-4d52-a87b-e409a4e981f2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:40.059754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37b9f496-8b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:40.094888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37b9f496-8b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:40.095594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:40.096156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:40.096544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:40.096930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37b9f496-8b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37b9f496-8b52-43c8-90fa-4e582241f062', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:5c:e7'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:40.137225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d33f27e8-2b02-45b1-87c3-d5d7de66c21c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:40.176438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e258f12a-507e-4465-aeac-ead7e694bae3]: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:40.176438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:40.176438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:40.176438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:40.176438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:40.176438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:40.176438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:40.176438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:40.176438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:40.176438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:40.176438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:40.176438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:40.176438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:40.177400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e258f12a-507e-4465-aeac-ead7e694bae3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:40.177792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:40.179733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:40.182799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[224ad83a-6a7b-4dad-8388-b815cdda6db9]: (4, ['qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:40.183646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:40.183970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:40.184228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:40.184564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:40.185701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[549df424-a842-4b52-8ce6-12d6916a318e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:40.186623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c447a94-6e5e-42b8-9d01-cccab515c779]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:40.187536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6616a44b-bbbc-4a1a-af06-14e61812f8ce]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:40.207682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[363902f9-8b2d-4538-9eac-b559b502b5ff]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 597620, 'reachable_time': 41440, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192618, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e102749-a79c-477f-b843-6b6c9616cf40]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:40.212514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-967313640-router", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135" Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:09Z", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6c:4f:08", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4924, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:13Z", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37b9f496-8b52-43c8-90fa-4e582241f062", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c369e92b-9b88-40b2-8c7d-00effae68044", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:5c:e7", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4929, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:08Z", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:37Z", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:40.215702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3 Jul 01 10:24:40.231058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 46a7f3d0-fe2b-4d54-bf16-e1d513957999. Wait time elapsed: 29.621 Jul 01 10:24:40.231058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 46a7f3d0-fe2b-4d54-bf16-e1d513957999. Wait time elapsed: 29.621 Jul 01 10:24:40.231058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:40.984899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.771s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:40.985512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:40.986256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:40.986668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:40.987013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:40.988300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[349f991e-8cdf-4abe-b801-f218371fed41]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.000337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5a405bb-2025-4edb-91a8-2485ef720a6a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.023922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29377e3d-838b-4436-bed8-32c96917ed9e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.050862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d56aa4f-7770-4b99-8227-f17fd0b680bb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192621, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.072050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7368809-5d38-4297-b421-11ac7162760a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.092494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a70159a5-921f-41a0-9e57-59b5916c0269]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.115138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3493ed7-dbe8-449f-95e7-037f9e023fc6]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.135606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d53c2850-4f06-4afe-813e-8cf77c54eaed]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.136764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:41.137043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:41.137308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:41.137606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37b9f496-8b52-43c8-90fa-4e582241f062) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:41.153302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:41.153553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c88ed2e2-d99f-4805-9ab1-25f99770ee15]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.156185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4ba668c-0544-408b-afbf-7e28c75ec72a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.157111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37b9f496-8b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:41.196237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37b9f496-8b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:41.197152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:41.197552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:41.198098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:41.198492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37b9f496-8b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37b9f496-8b52-43c8-90fa-4e582241f062', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:5c:e7'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:41.248489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fefcab0-3b41-4d11-873e-30d93203d81b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.282685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0f589a6a-00ae-4cc3-9fae-cf9ed93e9ec1]: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:41.282685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:41.282685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:41.282685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:41.282685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:41.282685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:41.282685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:41.282685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:41.282685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:41.282685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:41.282685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:41.282685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:41.282685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:41.283871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f589a6a-00ae-4cc3-9fae-cf9ed93e9ec1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:41.284364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:41.286177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:41.289469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a325c7ef-5e05-4623-b59d-e8d1625a64d3]: (4, ['qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.289469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:41.289469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:41.289469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:41.289469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:41.290177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5011e2d8-a43f-49b9-8b85-e42552af9ec4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.291226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[422ef66b-088d-4f55-8342-f7a3998ab326]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.292265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c3dd562-dac2-448f-ac19-731e9e9334ee]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.311851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[286ac3c5-f533-40fa-88a9-e8d5447b8a90]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 597731, 'reachable_time': 44292, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192640, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.317755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:41.318127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a2d1750-75ad-42a4-9fb7-c2554c88253c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:41.319002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-967313640-router", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135" Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:09Z", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6c:4f:08", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4924, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:13Z", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37b9f496-8b52-43c8-90fa-4e582241f062", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c369e92b-9b88-40b2-8c7d-00effae68044", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:5c:e7", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4929, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:08Z", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:40Z", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:41.324109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 46a7f3d0-fe2b-4d54-bf16-e1d513957999. Wait time elapsed: 30.728 Jul 01 10:24:41.336638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 2, update_id 46a7f3d0-fe2b-4d54-bf16-e1d513957999. Wait time elapsed: 30.728 Jul 01 10:24:41.336638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:24:45.199812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.878s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:24:45.200361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:24:45.201058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:24:45.201381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:45.201665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:45.202756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78924258-d027-4591-be32-ed0215ba58f4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.215198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f596d6e-276c-4ea4-aace-ec4940770f65]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.241859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e04824c0-1580-4a22-92de-d7d55e1e7c13]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.270442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2799c2e-047b-45b9-8a7d-e2e3e17babce]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 192705, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.291685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[350be6b8-707c-4bb9-a0d7-e62f5dfade81]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.312136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c19b8c08-ad68-4c3f-8c5c-65ddac82bd7e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.332988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[480a3ae3-cde6-4d57-b79a-c667adc85050]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.354745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[388e670b-af0f-4ca7-84cf-2e28e4f2d5e0]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.355642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:24:45.355924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:24:45.356223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:24:45.356509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37b9f496-8b52-43c8-90fa-4e582241f062) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:24:45.371874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:24:45.372207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e735743-7b9e-47e8-87c4-caeedc41539e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.374911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6aa8bb7e-d3c5-43a8-bc1b-99482e627311]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.375911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37b9f496-8b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:45.412596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37b9f496-8b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:45.413390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:45.413855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:45.415706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37b9f496-8b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:45.416337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37b9f496-8b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37b9f496-8b52-43c8-90fa-4e582241f062', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:42:5c:e7'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:24:45.479836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98c7d29c-32eb-4ee1-97c2-ed09fb3263ca]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.515455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[19ee3d25-fc51-42bc-8650-a342cf82d607]: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:24:45.515455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:24:45.515455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:24:45.515455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:24:45.515455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:24:45.515455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:24:45.515455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:24:45.515455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:24:45.515455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:24:45.515455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:24:45.515455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:24:45.515455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:24:45.515455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:45.516343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19ee3d25-fc51-42bc-8650-a342cf82d607]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:45.517345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:45.518951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:45.521701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[960f83d1-57cd-48d6-9b0e-f04492f0444e]: (4, ['qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.521701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:45.522529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:45.522849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:24:45.523298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:24:45.525653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c48f956e-fa74-4a4d-a903-bc8ff5f9933c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.526914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efb9178a-cddd-4924-acc1-f6c71332c15a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.528164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10b7a2ae-ebd6-4166-a047-fc0cf3c69821]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.551051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43bbdd02-f39e-416f-80ed-ecc3c8151827]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 598153, 'reachable_time': 20108, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 192723, 'error': None, 'target': 'qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.560635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:24:45.560887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65f22511-d4f8-41a6-bd2c-0751717a2a38]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37b9f496-8b not found in namespace qrouter-c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2. Jul 01 10:24:45.561926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-967313640-router", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135" Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:09Z", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "84da0ba8-3462-4b8c-b194-99caf0f15496", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:6c:4f:08", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4924, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.135", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:06Z", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:13Z", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37b9f496-8b52-43c8-90fa-4e582241f062", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "c369e92b-9b88-40b2-8c7d-00effae68044", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:42:5c:e7", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4929, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:08Z", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:42Z", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "2880b97f0771473e80554afa8df5a525", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0ad63df2-5214-4ead-a045-156fec415676", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:24:45.564608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3 Jul 01 10:24:55.265872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:25:07.821242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-18eee1c8-6c7c-487f-85ba-7bac82099593 tempest-TestMinimumBasicScenario-2052601014 tempest-TestMinimumBasicScenario-2052601014-project] Got routers updated notification :['88bbc4fa-c162-4343-9551-ceded0fe88c3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:25:07.822689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id 208437c8-a81a-4fa5-b967-e03096802ec2. Wait time elapsed: 0.001 Jul 01 10:25:07.823187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id 208437c8-a81a-4fa5-b967-e03096802ec2. Wait time elapsed: 0.001 Jul 01 10:25:07.823746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:25:08.704346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.880s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:25:08.704609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:25:08.705707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:25:08.706033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:08.706312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:08.707641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43230d2c-7400-4b7f-9c27-ebb38d718401]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:08.733405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9b387a2-1bd5-4e18-a3f7-5c537284aac4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:08.760694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5430b18e-01ec-471d-afac-06e6ca9e41fa]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:08.795199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[273c174f-0083-4932-aad8-bc34616891d7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 193636, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:08.821110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22d37cc6-e968-4263-bb8d-617d5733402f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:08.846993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0cab6cd-9374-41aa-af6a-3c02f3ce232a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:08.869765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66a94dd7-3a37-4314-9be0-79a541b69348]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:08.896191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81c44ece-222e-4a25-b514-2ea993abc2f5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:08.896777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:08.897115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:08.897392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:25:08.897675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:25:08.920240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:25:08.920805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[804ee9a2-902c-431c-854c-1099748a4978]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:08.923770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69c19902-b40b-4fe2-870e-ec927c189a34]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:08.925001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:08.968979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:08.971794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:08.974610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:08.979775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:08.980457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:09.034238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[182119d7-4e58-4d71-a978-5e2fbdb6adba]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:09.065262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b56eae28-269e-4698-915f-fa76aff79a0e]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:25:09.065262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:25:09.065262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:25:09.065262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:25:09.065262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:25:09.065262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:25:09.065262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:25:09.065262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:25:09.065262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:25:09.065262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:25:09.065262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:25:09.065262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:25:09.065262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:09.066258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b56eae28-269e-4698-915f-fa76aff79a0e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:09.066504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:09.068417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:25:09.070685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[072c95bd-cada-4819-8ffb-1931a4284b63]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:09.070685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:09.070685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:09.070685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:25:09.070685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:09.072096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33112d3b-523c-4ba6-9031-56ee9b62d4df]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:09.073086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d938bc4-8ac3-4fdf-af5e-b7828ef42aee]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:09.073999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9141756-602e-4a3a-8180-98c936e0c4cc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:09.095291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9397cfa-dfff-4efd-9f61-0c709a31d4df]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 600505, 'reachable_time': 15000, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 193655, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:09.101650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:25:09.101977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e611156f-f246-4186-9fe3-ffeedc97ebb0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:09.103572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f6b1d88c-9343-4c03-bd4f-94aa5a1c3954", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.69", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "138f59b0-bc80-4872-a1c7-d79903f9fac4", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4968, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d6:ea:25", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b0a75819-bfa7-4556-a56a-6c4cdd215a5d", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:25:06Z", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:25:06Z", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:25:09.106378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:46Z", Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 208437c8-a81a-4fa5-b967-e03096802ec2. Wait time elapsed: 1.283 Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 208437c8-a81a-4fa5-b967-e03096802ec2. Wait time elapsed: 1.284 Jul 01 10:25:09.123298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:25:12.984365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.878s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:25:12.984969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:25:12.985632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:25:12.985903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:12.986183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:12.987332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d8b069f-bd76-412c-9c9e-9de22775b677]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.005777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[896e5ceb-f735-4dfb-be48-debea33ee845]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.027338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e39524b2-e2e0-4609-a3fb-ef188f897a85]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.056817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[680aeddd-e548-4c2b-98d3-3cc59a363578]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 193722, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.077306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a62e4828-957d-4b79-af6b-1a82220e25f8]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.098909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8f41292-1ebd-482c-9798-107ed9a2b040]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.120476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56bb546a-91df-49c7-a22b-0e265f2568e7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.142516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[200ab295-5373-4059-b259-a42c092c4296]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.143081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:13.143376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:13.143657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:25:13.143938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:25:13.160971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:25:13.161269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1d1f3d7-59b3-4a07-b158-3ddf1eddb9f4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.165065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[746d14db-2821-49c9-827a-d8ee7623e5d1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.166190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:13.203072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:13.203851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:13.204332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:13.204779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:13.205224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:13.262031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11210795-dea1-4356-81bf-30b2e0886538]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.297951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f39379c1-281a-4289-a6e8-8a11efe2b0e7]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:25:13.297951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:25:13.297951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:25:13.297951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:25:13.297951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:25:13.297951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:25:13.297951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:25:13.297951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:25:13.297951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:25:13.297951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:25:13.297951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:25:13.297951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:25:13.297951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:13.298970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f39379c1-281a-4289-a6e8-8a11efe2b0e7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:13.299188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:13.302024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:25:13.304776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef98a1c0-ddae-4352-a7ef-3347c8e4b997]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.304776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:13.305131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:13.305651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:25:13.305983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:13.307020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6cb3bfff-3ec5-468d-a939-b456fc51cbf6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.307997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec1fd132-66d3-4d3d-8e80-69f1970e60c8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.308935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16c2cc9a-c4d2-42a2-b4ea-1cca7a3b9130]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.330371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22fd4fb0-4d38-4396-82a7-0a94c3c4a36d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 600932, 'reachable_time': 32060, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 193740, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.338553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:25:13.338917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bcf0e87-415c-4bcc-926c-8eb0ab0c75c8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:13.340314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f6b1d88c-9343-4c03-bd4f-94aa5a1c3954", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.69", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "138f59b0-bc80-4872-a1c7-d79903f9fac4", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4968, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d6:ea:25", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b0a75819-bfa7-4556-a56a-6c4cdd215a5d", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:25:06Z", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:25:06Z", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:25:13.343180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:25:10Z", Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 208437c8-a81a-4fa5-b967-e03096802ec2. Wait time elapsed: 5.520 Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 208437c8-a81a-4fa5-b967-e03096802ec2. Wait time elapsed: 5.520 Jul 01 10:25:13.356217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:25:14.197064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.854s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:25:14.197692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:25:14.198841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:25:14.199164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:14.199469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:14.200602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9402a820-3f64-485c-b052-124c0b903afd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.212646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5b195bc-3523-45ca-9870-c47c28a742dd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.234466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e824441-6029-46a9-b9ef-43f428f19268]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.270268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6db215d4-8d66-4999-a640-6627351e774c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 193743, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.292050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5f7cd6e-21bd-4982-bd76-4137a41fdb4f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.312227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1eee75d-4061-4544-962d-58cbf4e3fab4]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.333711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69eadcef-2cf3-42bb-b788-eff82a42db48]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.356392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f82eb448-f6d9-4d28-a530-d1f1f8f75690]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.357066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:14.357405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:14.357736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:25:14.358080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:25:14.374133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:25:14.374493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b42373ee-9b49-4769-a2d8-3a07d8247c35]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.377257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97be13fa-8b8a-490e-973e-0223a51fad4c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.378446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:14.421245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:14.421245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:14.421245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:14.421245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:14.421245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:14.476733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08c0dfe0-d8f5-45a8-8343-fd8717cc4732]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.507085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[42e1a667-ee9e-4c36-a7d1-29daa8ffe186]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:25:14.507085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:25:14.507085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:25:14.507085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:25:14.507085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:25:14.507085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:25:14.507085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:25:14.507085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:25:14.507085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:25:14.507085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:25:14.507085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:25:14.507085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:25:14.507085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42e1a667-ee9e-4c36-a7d1-29daa8ffe186]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:14.509450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:14.511504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:25:14.515863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce8e59e1-aab8-4640-a02a-f1928358c713]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.515863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:14.515863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:14.516139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:25:14.517351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:14.517351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41ce607e-3336-4701-aed3-99e5e6f4c0e5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.518439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f342f0b-5625-457c-8dee-37cffc7a164b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.519594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[595c431b-8392-4a1a-b6cc-14101f9adea2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.540771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47e09197-2d05-477a-a746-2c06a202ae8b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 601053, 'reachable_time': 30204, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 193762, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.547202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:25:14.547541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16287209-903e-4e50-81f9-eb3b996c7c8e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:14.548647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f6b1d88c-9343-4c03-bd4f-94aa5a1c3954", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.69", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "138f59b0-bc80-4872-a1c7-d79903f9fac4", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4968, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d6:ea:25", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b0a75819-bfa7-4556-a56a-6c4cdd215a5d", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:25:06Z", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:25:06Z", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:25:14.551788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:25:13Z", Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 208437c8-a81a-4fa5-b967-e03096802ec2. Wait time elapsed: 6.729 Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 208437c8-a81a-4fa5-b967-e03096802ec2. Wait time elapsed: 6.730 Jul 01 10:25:14.564601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:25:15.141324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:15.141557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:15.141875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:15.142095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:17.638497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.086s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:25:17.639154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:25:17.640228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:25:17.640553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:17.640835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:17.642489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc1c7791-7546-4052-bd1f-c66a903aaa06]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.656015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfe34501-e2ec-4efb-bec4-a727fdd36425]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.677933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a62d3ed8-c453-4023-ba36-2bcbf1cddec8]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.706227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66c7dd8c-6aa7-484e-b23b-8691ad17668d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 193772, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.728078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8aedabe8-13ab-435d-8926-e01d1dc4f030]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.749857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0dc72ad8-73bf-4ddd-ba48-5a632f3d64de]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.771497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee2c208c-1dc3-4753-bc63-c5b55af45a67]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.797211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee4ad8a8-b976-4b5c-b4a5-b2fd70bdb5f6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.797788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:17.798071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:17.798344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:25:17.798625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:25:17.813163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:25:17.813542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[873bfd7b-0af6-4d93-82d3-c6a66d59d2af]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.816374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a81c1095-5d09-4b55-9c19-84b5394e6e11]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.817433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:17.858415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:17.859086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:17.859513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:17.859925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:17.860309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:17.912162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d1ba0d9-0fdd-4ddc-9757-be54cfefb048]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.943469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[03f1e467-fa0a-4097-a767-85921e9b5348]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:25:17.943469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:25:17.943469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:25:17.943469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:25:17.943469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:25:17.943469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:25:17.943469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:25:17.943469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:25:17.943469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:25:17.943469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:25:17.943469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:25:17.943469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:25:17.943469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03f1e467-fa0a-4097-a767-85921e9b5348]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:17.944679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:17.949809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:25:17.952397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce5a60b3-238d-4721-9d0e-fbdc1e5e8009]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.952397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:17.952720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:17.952720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:25:17.953085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:17.954101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86caf053-b984-401a-9ca7-889f0114b4d1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.955117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a4bc92a-8861-449f-b837-9f2e5472e33d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.956430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[444b391a-4124-4adb-9570-e37455aeaea5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.980257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[845f87cd-18ab-4c9d-b028-b196f452d029]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 601396, 'reachable_time': 16296, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 193791, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.982517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:25:17.982947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4acb3be6-062b-43ca-aad9-c03b1ad8f31a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:17.984113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f6b1d88c-9343-4c03-bd4f-94aa5a1c3954", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.69", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "138f59b0-bc80-4872-a1c7-d79903f9fac4", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4968, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d6:ea:25", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b0a75819-bfa7-4556-a56a-6c4cdd215a5d", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:25:06Z", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:25:06Z", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:25:17.988318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:25:15Z", Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 21, Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 208437c8-a81a-4fa5-b967-e03096802ec2. Wait time elapsed: 10.167 Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 208437c8-a81a-4fa5-b967-e03096802ec2. Wait time elapsed: 10.168 Jul 01 10:25:18.006377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:25:18.974444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.983s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:25:18.976048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:25:18.977352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:25:18.978861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:18.979344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:18.984489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9e8ce18-253c-40a3-8489-85df881b7c16]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.028653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f62eb04c-c5a0-487d-b8ad-8205567ee6e3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.077882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85c65e6b-9c8d-460a-af97-4894df08c4a5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.142914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e89dd39d-1d0a-470b-9bb2-3dc12fb10ef0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 193853, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.165371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[830580a1-f79a-41ce-a0fd-9cee0cd05962]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.191810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de2ee50d-4c97-42f4-8408-488acb18b4c6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.218002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f09614ac-3874-4088-b76a-b8daf51fff7d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.240132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d04357f-e345-4ae0-b8fa-69d96fd29d1a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.240627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:19.240905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:19.241172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:25:19.241454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:25:19.256489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:25:19.256778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9910894f-93dc-4e0d-adef-a7401d150bf1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.260810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24a1d167-cc56-487d-b5e0-c99878d1fbd6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.261747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:19.290815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:19.291728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:19.292322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:19.292941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:19.293416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:19.344457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c074c6a8-de44-46c6-bcf8-3c0c476a6be3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.376895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[93d862f4-2c1a-4c58-9c33-ff5d545b081c]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:25:19.376895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:25:19.376895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:25:19.376895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:25:19.376895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:25:19.376895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:25:19.376895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:25:19.376895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:25:19.376895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:25:19.376895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:25:19.376895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:25:19.376895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:25:19.376895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:19.377939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93d862f4-2c1a-4c58-9c33-ff5d545b081c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:19.378735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:19.380675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:25:19.383423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7fa06aa-eaa6-4ce2-a79f-760d564e0282]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.384273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:19.384651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:19.384901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:25:19.385458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:19.386564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbfd50ed-7530-4c13-b24d-d983adc26ad3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.387571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44ccde92-c61d-434f-a335-c454b4ef452b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.388649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcfc1b0f-2269-4151-9241-262caaf8f3fe]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.411152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1843ee53-d96b-46e3-8bef-e6194a5fda9d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 601540, 'reachable_time': 37052, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 193873, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.413435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:25:19.414391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d55de645-d3a8-4dc9-99cd-06c368653a60]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:19.415616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f6b1d88c-9343-4c03-bd4f-94aa5a1c3954", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.69", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "138f59b0-bc80-4872-a1c7-d79903f9fac4", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4968, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d6:ea:25", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b0a75819-bfa7-4556-a56a-6c4cdd215a5d", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:25:06Z", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:25:06Z", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:25:19.420000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:25:17Z", Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 208437c8-a81a-4fa5-b967-e03096802ec2. Wait time elapsed: 11.599 Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 208437c8-a81a-4fa5-b967-e03096802ec2. Wait time elapsed: 11.599 Jul 01 10:25:19.430472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:25:22.292371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.871s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:25:22.293034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:25:22.294058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:25:22.294388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:22.294671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:22.295849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4f94646-4c04-41b7-a827-2b842e9409e1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.309398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8fdd4112-45ec-4e68-844c-ce0598c15467]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.334850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a203081-feca-4f3b-a47c-9fc6d993bb6d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.362195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74c69ece-3883-4187-b441-c033fe40a07c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 193912, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.384913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4d2cf13-8a77-473a-b142-985b8fb4a0db]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.405395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8081871f-cba1-4a86-9f79-471df5e927d2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.429795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c546da49-765b-47dc-b052-164811416633]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.451252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2419a37f-b2fc-4e03-9fe9-9b2ae2df50bd]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.451838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:25:22.452116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:25:22.452386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:25:22.452664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:25:22.467704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:25:22.467999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[978f1518-c10a-4825-84a7-583c8f9d25d3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.470595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c027381b-f569-44df-84cc-6a0c6585674c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.471561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:22.519693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:22.520375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:22.520807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:22.521283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:22.521681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:25:22.584627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21367466-6971-42a6-8311-fbef6268eef1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.618245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ee0b2537-6ae4-4836-a7b8-0d29079f6635]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:25:22.618245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:25:22.618245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:25:22.618245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:25:22.618245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:25:22.618245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:25:22.618245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:25:22.618245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:25:22.618245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:25:22.618245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:25:22.618245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:25:22.618245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:25:22.618245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:22.619442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee0b2537-6ae4-4836-a7b8-0d29079f6635]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:22.621565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:22.623674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:25:22.626429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79cce94d-473f-42c7-9a80-f93113264f26]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.627601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:22.628015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:22.628536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:25:22.628989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:25:22.630071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b27cd8a-dc55-41d9-90b1-945235fe8346]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.631137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f75aad6-0ad2-4c8c-a682-faaf0bfc392f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.632668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1c76c06-3591-4665-b6f3-64cd93b5f4c0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.654441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a3500d6-6f2f-43d3-9a5e-91dd8a317f3a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 601862, 'reachable_time': 20172, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 193930, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.661920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:25:22.662326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2a0018a-0244-46e4-ba79-338c5176a383]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:25:22.663576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f6b1d88c-9343-4c03-bd4f-94aa5a1c3954", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.69", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "138f59b0-bc80-4872-a1c7-d79903f9fac4", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4968, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d6:ea:25", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b0a75819-bfa7-4556-a56a-6c4cdd215a5d", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:25:06Z", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:25:06Z", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:25:22.667432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:25:21Z", Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 26, Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:25:22.682698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3 Jul 01 10:25:35.267567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:26:15.142160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:26:15.142797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:26:15.142797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:26:15.142979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:26:15.269641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:26:55.270638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:27:15.143177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:27:15.143859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:27:15.143859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:27:15.144045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:27:35.272398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:28:15.143909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:15.144633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:15.144633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:15.144862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:15.274102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:28:42.985170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-2e427c72-97da-4133-8950-cdec8d1372eb tempest-TestMinimumBasicScenario-2052601014 tempest-TestMinimumBasicScenario-2052601014-project] Got routers updated notification :['88bbc4fa-c162-4343-9551-ceded0fe88c3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:28:42.986916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id fb00de8f-7437-409f-bf7c-d2c1c873536c. Wait time elapsed: 0.001 Jul 01 10:28:42.987338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id fb00de8f-7437-409f-bf7c-d2c1c873536c. Wait time elapsed: 0.001 Jul 01 10:28:42.987656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:28:43.829747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.842s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:28:43.830696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:28:43.831704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:28:43.832978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:43.833277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:43.837953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3937096b-dc88-4228-b625-c53bdc27d3e7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:43.851846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46d3f2b9-dab6-4e2c-b58f-d1758b121a9a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:43.873312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[881c67ca-680f-4621-aa25-84e856aefc20]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:43.903658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9c4ebfd-5435-4881-bd5f-17395942373f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 197224, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:43.925421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5dce909e-a157-4d74-bea5-bf83580a03bd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:43.947848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc002bdb-f18b-4aa4-bc11-6390072310ad]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:43.968859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04504f05-2070-43ed-b54d-472f09b2af08]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:43.990132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c3cbea2-e455-42ea-a611-6fb8b0aa30ba]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:43.991558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:43.991558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:43.991558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:28:43.991866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:28:44.011071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:28:44.011335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[193b80fe-6c35-42c3-9fd8-98aefb5757cc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:44.014041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4371f784-c6aa-4490-9b29-a670e8eb50cf]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:44.015008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:44.058768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:44.060121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:44.060739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:44.061330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:44.062160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:44.132282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b234d42-0892-463b-9b05-fc34b15c385e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:44.181833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b2949d36-d052-4a2c-8fad-e59631a89b25]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:28:44.181833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:28:44.181833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:28:44.181833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:28:44.181833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:28:44.181833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:28:44.181833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:28:44.181833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:28:44.181833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:28:44.181833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:28:44.181833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:28:44.181833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:28:44.181833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:44.182997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2949d36-d052-4a2c-8fad-e59631a89b25]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:44.183114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:44.185537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:28:44.189395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4bb5ab56-dbf8-4369-9fd6-c2905182f9c9]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:44.189395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:44.189395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:44.189395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:28:44.189395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:44.197397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69da6e14-f8c9-463e-893a-13e1923ace59]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:44.198842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da3a3440-152e-407a-b3a7-a2eddb79fdc0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:44.200045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e03d5c43-a5a1-4f89-a5e7-3135ba0175f3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:44.235203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[895b43b7-53c8-472f-8a4a-5a0b31ed9431]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 622016, 'reachable_time': 31176, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 197243, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:44.243341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:28:44.243661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6bd548f-7972-4707-bb53-18d6e72994f7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:44.244928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:25:24Z", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:28:44.249163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:28:44.266208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id fb00de8f-7437-409f-bf7c-d2c1c873536c. Wait time elapsed: 1.272 Jul 01 10:28:44.266208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id fb00de8f-7437-409f-bf7c-d2c1c873536c. Wait time elapsed: 1.272 Jul 01 10:28:44.266208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:28:47.524476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.266s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:28:47.528010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:28:47.530261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:28:47.530261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:47.530261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:47.536413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e233f122-d11f-4322-821c-9578c46e9c15]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.549705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b732686d-1998-43a1-a794-dad41da81967]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.579322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59360ea5-ab0e-4eed-afb3-43d1f8601b89]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.637628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e9dc632-6777-448c-a889-2defd2ea91ec]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 197359, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.677476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f1f3130-7cc8-4192-bc37-ea7cd98c3ca3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.708309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b945d9d-c979-4977-8f8e-7ecb6b7ade37]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.734280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c10634c5-7680-451b-957f-0c10abc6a3bd]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.757383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f069b4fb-0e1a-4c07-8b20-8440efb82b14]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.757970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:47.758261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:47.758721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:28:47.759261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:28:47.775277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:28:47.775672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[281cc765-2947-4819-a4d7-432bc069f4a0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.778465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8830e427-b1aa-49f4-a75f-aa363b488b1f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.780054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:47.827189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:47.828175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:47.828733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:47.829281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:47.829972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:47.880217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[027796f4-8363-474d-ad11-778a9295292f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.912112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[938daff1-097e-4467-9963-4d3385596a7b]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:28:47.912112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:28:47.912112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:28:47.912112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:28:47.912112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:28:47.912112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:28:47.912112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:28:47.912112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:28:47.912112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:28:47.912112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:28:47.912112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:28:47.912112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:28:47.912112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:47.913789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[938daff1-097e-4467-9963-4d3385596a7b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:47.917173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:47.919128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:28:47.924684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5104889-1ed7-4051-8b72-fd6a5fc71355]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.924684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:47.924684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:47.924684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:28:47.924684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:47.924684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29b93f6e-5e65-48f2-8fe6-6e37169d25f0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.924684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13ec3ce6-433e-4b03-a4e7-0bd7b39c7b82]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.924684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eda92195-dd98-4174-b396-87231056bb38]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.961581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[074a7222-add4-4ff8-9166-8eb975abbc35]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 622390, 'reachable_time': 36640, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 197381, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.970095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:28:47.970420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bdc927b-6482-4eb4-9ce6-f4a591c2ffc0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:47.971611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:25:24Z", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:28:47.974628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id fb00de8f-7437-409f-bf7c-d2c1c873536c. Wait time elapsed: 4.987 Jul 01 10:28:48.003548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id fb00de8f-7437-409f-bf7c-d2c1c873536c. Wait time elapsed: 4.987 Jul 01 10:28:48.003548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:28:48.828389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.855s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:28:48.829105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:28:48.829763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:28:48.830295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:48.830822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:48.832740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3491189f-e9f8-4fa4-8d1c-1e009fbbe2c6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:48.850766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dadebddd-8958-4b39-a939-3012f0aaf7bc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:48.873915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[065395af-beb5-47c5-a351-138e4a169eb2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:48.922010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70e2d75a-1bde-4f96-ba49-f32e0ee251f0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 197390, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:48.949557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdc35439-351e-4ebe-ba79-da0f91e14d74]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:48.972168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ba27a28-d5db-44cf-89e9-7a6c1fe25d75]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:48.998222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd5bc77e-5244-4230-a63d-5f948044e5f0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:49.025946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90c5884f-3fb1-4056-b512-a6b15f301207]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:49.027217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:49.027217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:49.027549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:28:49.027933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:28:49.046606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:28:49.047200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92bbfaed-b51b-4931-b9f2-9d45e3242c41]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:49.050778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd0c646f-c986-4584-8c83-70b8fcae447a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:49.052050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:49.097105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:49.098750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:49.099592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:49.100654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:49.101227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:49.163616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[231d0ac5-f333-4043-b6f0-f8146f6646e3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:49.218338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a9cc0a27-0ff6-422f-ae92-1210046cd00c]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:28:49.218338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:28:49.218338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:28:49.218338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:28:49.218338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:28:49.218338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:28:49.218338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:28:49.218338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:28:49.218338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:28:49.218338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:28:49.218338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:28:49.218338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:28:49.218338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:49.219444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9cc0a27-0ff6-422f-ae92-1210046cd00c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:49.220328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:49.222224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:28:49.226327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03e700bd-06bb-40dd-8668-2cd2cf52adb7]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:49.226327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:49.227506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:49.227506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:28:49.228357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:49.229552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3071c96-b63e-4c4b-9e87-ac80a55e5eb6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:49.230894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4b99ed0-b5e6-4fac-b827-6557b2a0a3a1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:49.232401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8ad4ef5-8ade-4b29-a361-49d4dc957062]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:49.257888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abcc7cbe-6e74-4994-bd6f-05697a910767]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 622517, 'reachable_time': 16988, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 197411, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:49.259980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:28:49.260403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74977e89-2385-4db0-87a0-94365e02c656]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:49.261698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:28:46Z", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 30, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:28:49.268369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:28:49.281929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id fb00de8f-7437-409f-bf7c-d2c1c873536c. Wait time elapsed: 6.283 Jul 01 10:28:49.281929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id fb00de8f-7437-409f-bf7c-d2c1c873536c. Wait time elapsed: 6.283 Jul 01 10:28:49.281929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:28:51.890083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.620s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:28:51.890683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:28:51.891523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:28:51.891916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:51.892264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:51.894317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55c48600-fee5-4200-9bf7-bcbf3e098c84]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:51.910712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b300996-5d40-4b98-b9ae-18b4ce4e04c9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:51.931815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa81f55c-8b0e-4901-8c6e-996f4d16c128]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:51.965040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[819e02bc-dad9-4f35-81aa-5791080e39f6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 197430, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:51.987612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b417b56-2c83-4b88-ab20-e008cb5326e8]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.013401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eac95b90-67d5-4528-b1ff-1302297f769f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.039537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21bb9dff-6a15-4840-8067-a3865b240ad1]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.089457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04e22514-27d4-4328-ad9f-23dbdf726eef]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.090387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:52.090677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:52.090948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:28:52.091228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:28:52.128038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:28:52.128507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b67566ec-71a0-456b-822c-e1c8955ff95b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.132075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc9d923d-cccd-4b86-8dfc-07c6bd9ae4e6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.133269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:52.165311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:52.166238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:52.166958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:52.168137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:52.168782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:52.238758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[107527ab-5737-43f9-85af-d1454de9aa27]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.288384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[6f93b60f-63cc-40f5-92c6-f866c8f455f3]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:28:52.288384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:28:52.288384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:28:52.288384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:28:52.288384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:28:52.288384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:28:52.288384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:28:52.288384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:28:52.288384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:28:52.288384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:28:52.288384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:28:52.288384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:28:52.288384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:52.289587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f93b60f-63cc-40f5-92c6-f866c8f455f3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:52.290191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:52.292784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:28:52.296632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[937f9492-d28d-4e9a-ac77-a974c4e7a97b]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.296794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:52.297211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:52.297678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:28:52.298244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:52.299652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4dcbcc65-3171-4e0d-9a6f-bcf0ed1b4f05]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.300718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[679e6270-ef27-402d-b87a-82fd84ece317]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.301729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdc6de54-e1cc-4bb8-8a4c-922e0f7d11f5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.325049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f5fdc29-bea2-49b2-8ca1-0647f9905a11]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 622822, 'reachable_time': 38244, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 197450, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.332652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:28:52.332964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ea5d974-1f6f-4192-b10d-8e0b60140e05]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:52.334028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:28:49Z", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 32, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:28:52.341359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id fb00de8f-7437-409f-bf7c-d2c1c873536c. Wait time elapsed: 9.351 Jul 01 10:28:52.360519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id fb00de8f-7437-409f-bf7c-d2c1c873536c. Wait time elapsed: 9.351 Jul 01 10:28:52.360519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:28:53.170848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.833s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:28:53.171444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:28:53.172157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:28:53.172809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:53.173095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:53.179897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36984dd9-20af-4761-b0eb-46b67d1cb4a4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.204715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61c0f7d0-9370-43f8-bb28-a8882127684b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.243179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e41e33b-ef14-46d7-868d-2c7b4949b4df]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.290894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9a9184f-e4f2-4533-8ef3-7351970621ef]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 197462, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.344336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3643595-b756-453b-9535-b2ed698054c8]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.382608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e70bc01-3dc3-4aa8-bf74-607d8aea1f65]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.444448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[324db7bc-3800-4a32-b43a-fc4741c1a0bc]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.528825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11ad98ba-383f-4127-9472-30f72c1a1964]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.529456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:53.529689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:53.529963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:28:53.530246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:28:53.588168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:28:53.590382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ba903c1-bf90-4308-898f-781e7139f4e3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.598965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f8e95dc-e84e-4408-baf5-a77aea9b0cf6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.609965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:53.706282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:53.707070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:53.707619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:53.708273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:53.708772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:53.851258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48c3c1bf-0f78-4265-a2a1-24722952df90]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.903457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9471960c-9869-428e-88dd-e0f9c2d904e4]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:28:53.903457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:28:53.903457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:28:53.903457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:28:53.903457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:28:53.903457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:28:53.903457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:28:53.903457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:28:53.903457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:28:53.903457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:28:53.903457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:28:53.903457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:28:53.903457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:53.907001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9471960c-9869-428e-88dd-e0f9c2d904e4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:53.907728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:53.910307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:28:53.919342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55895cf8-b675-47a9-9d69-663a12e1e9b5]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.920369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:53.920805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:53.921271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:28:53.921872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:53.923222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33ee1d16-5dd1-4352-98ca-e13abd5e4eb5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.929283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f58398f-f55b-4e87-bf9a-c9adba9d5222]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.930399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7254fdc6-6868-4c37-aa42-96d719f599f9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.952369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[846283d8-3725-48a3-85f8-60de919510ca]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 622955, 'reachable_time': 37564, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 197539, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.957600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:28:53.957600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44a9c797-749f-4077-b684-96dd929b95e2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:53.958422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:28:51Z", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 34, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:28:53.964888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:28:53.979770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id fb00de8f-7437-409f-bf7c-d2c1c873536c. Wait time elapsed: 10.984 Jul 01 10:28:53.979770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id fb00de8f-7437-409f-bf7c-d2c1c873536c. Wait time elapsed: 10.985 Jul 01 10:28:53.979770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:28:55.275688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:28:56.616478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.640s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:28:56.617767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:28:56.618783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:28:56.619315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:56.619717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:56.621336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8a4dead-49f0-41f7-b6aa-0178d7c368ae]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.637331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7452832d-0a8d-4e84-aa5b-0cab13de1f67]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.663095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b47ed575-027e-4443-b9de-dc0af2a81b17]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.709946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86f62abf-213f-4766-9923-8c5179637e8a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 197554, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.735302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ad58120-6afc-46c3-a7ae-23530f072cd4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.773398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[afcc505c-91ae-44c2-8f42-9591be6a31c7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.803182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4c65ee8-7aa1-4894-b0b2-db0de3edd492]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.833030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55bd51cc-13d7-4059-9523-29a2e53f3179]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.833608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:28:56.834731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:28:56.835084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:28:56.835399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:28:56.855055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:28:56.855606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[537336d9-5e16-4643-b1ff-ea6b5cac62ad]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.859143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe4d52b5-7f31-49f6-8b53-98798f10805f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.860410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:56.899650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:56.900297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:56.900749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:56.901136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:56.901739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:28:56.948687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b634cd0-98d9-4611-8359-87d9555b13a3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.978730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[791baccd-6eb5-4549-8e3e-929249eee0f1]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:28:56.978730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:28:56.978730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:28:56.978730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:28:56.978730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:28:56.978730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:28:56.978730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:28:56.978730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:28:56.978730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:28:56.978730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:28:56.978730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:28:56.978730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:28:56.978730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:56.979968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[791baccd-6eb5-4549-8e3e-929249eee0f1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:56.980628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:56.982993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:28:56.985709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54008f64-515a-43fb-b15e-ce2ee4e138cd]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.985917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:56.986307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:56.986565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:28:56.986902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:28:56.987962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30fec863-918c-4455-b0d7-595209100db1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.988959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98de9977-a4ff-4f9d-80a4-60e5fd238b15]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:56.989919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06d8313d-6b3c-4018-8d29-d7870cdd5b4a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:57.011149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[362ff4bb-d3d6-493c-9dcd-54bea2413fef]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 623297, 'reachable_time': 23872, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 197572, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:57.019065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:28:57.019583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e31f4eb6-a6dc-461e-95eb-208ec9545c39]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:28:57.020681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:28:53Z", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 36, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:28:57.024319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3 Jul 01 10:29:15.144555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:29:15.145722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:29:15.147198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:29:15.147683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:29:35.277690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:29:49.878449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-94363aee-5298-4510-965b-f7b9de0221c3 tempest-TestMinimumBasicScenario-2052601014 tempest-TestMinimumBasicScenario-2052601014-project] Got routers updated notification :['88bbc4fa-c162-4343-9551-ceded0fe88c3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:29:49.879625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id cc07547b-2350-438c-a71d-8175a1d653e5. Wait time elapsed: 0.001 Jul 01 10:29:49.880587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id cc07547b-2350-438c-a71d-8175a1d653e5. Wait time elapsed: 0.001 Jul 01 10:29:49.880981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:29:51.120150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.239s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:29:51.120703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:29:51.121534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:29:51.121926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:29:51.122269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:29:51.123518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75b6d94e-d283-4687-a61e-9a0180cd6ac8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.140912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fb674ce-7f08-47df-a1f9-1627e54c8cb4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.164882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ebb1e8e4-b448-46a4-9c03-ddfbc08dba9e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.200974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0136907f-45eb-42c3-b3b2-814fed2105d7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 198447, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.222602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87419b6f-017e-4db1-8abc-270bee2582bc]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.244754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[daf90f29-5881-4da6-8b80-22d4e2c5d0d2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.267306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea1a7d1a-0d55-4048-a0e7-828407496bda]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.289092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4147cdfa-93a0-46d4-954e-d68180c77456]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.289763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:29:51.290110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:29:51.290443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:29:51.290912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:29:51.307763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:29:51.308143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d91cae5-d9cc-43f3-b1b6-996474c47f46]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.311208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34cb9ce2-c1cf-4d56-8632-9f05391d49a6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.312319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:51.369877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:51.371524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:51.372282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:51.373039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:51.373803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:51.429408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70fad133-2a83-43b0-857b-c60641a48894]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.461075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[573e38b3-ba99-4d51-aad1-5e5ae570456d]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:29:51.461075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:29:51.461075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:29:51.461075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:29:51.461075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:29:51.461075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:29:51.461075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:29:51.461075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:29:51.461075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:29:51.461075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:29:51.461075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:29:51.461075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:29:51.461075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[573e38b3-ba99-4d51-aad1-5e5ae570456d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:51.462335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:51.464737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:29:51.469918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50d9e4bc-bf48-4568-b93c-b99a0d39de41]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.469918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:29:51.469918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:29:51.469918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:29:51.469918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:29:51.469918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[753edeed-12b5-4daa-9678-ee3a3d5847aa]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.471173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e31a1c1f-8ad2-4d52-bce8-9f33c81b1f48]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.472367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e50533bd-0670-419d-8b18-57af7df5bc3f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.493153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0be9980f-3207-40f0-9c03-9815f7f28c93]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 628746, 'reachable_time': 40788, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 198466, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.505389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:29:51.506393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1306cd84-793d-47fa-ae24-07ca8a6535b1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:51.507602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6deb3207-20b8-427e-bc26-09675ad7cd79", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.172", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "77850af0-a181-4d55-94e9-66b48525994d", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5244, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bf:7e:06", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "207c801c-3a6a-4381-9136-8c987e421fe4", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:29:49Z", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:29:49Z", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:29:51.511523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:29:00Z", Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 41, Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id cc07547b-2350-438c-a71d-8175a1d653e5. Wait time elapsed: 1.634 Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id cc07547b-2350-438c-a71d-8175a1d653e5. Wait time elapsed: 1.634 Jul 01 10:29:51.529633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:29:55.641167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.127s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:29:55.641809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:29:55.642469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:29:55.642860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:29:55.643210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:29:55.644785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f4d91e0-11ec-4e3c-83eb-4bbf065f89ee]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.664544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d863f06e-d97f-4e87-8c51-6bf4c923fa79]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.686034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20124aca-da01-4706-9cd6-fa002a21e1e7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.715833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab7cde75-ef86-47fe-8f32-0940a5f3fea8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 198532, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.736536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e31ee6e-949a-419e-a185-794d553ebe43]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.758542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd5c9617-bd23-45b8-979f-9771d8f579e1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.780871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db6bd914-82a1-437a-81d2-38b8dfa4a21d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.802728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ebd4a0c8-ef71-4440-a4e5-4ca9e6f0d0ac]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.803718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:29:55.803976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:29:55.804637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:29:55.806003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:29:55.822854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:29:55.823668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8023f929-c852-4a1e-9d91-96a29751944a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.826824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[370ddad9-e8e6-47f6-8389-1a9db4c41d4c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.828021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:55.873515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:55.873515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:55.873515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:55.873515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:55.873515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:55.920778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6baf6d4-c360-4ab4-9a91-003e7894cba2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.964636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f56c4c4b-c80c-418c-80d8-6a18f33b751a]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:29:55.964636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:29:55.964636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:29:55.964636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:29:55.964636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:29:55.964636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:29:55.964636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:29:55.964636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:29:55.964636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:29:55.964636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:29:55.964636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:29:55.964636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:29:55.964636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:55.965766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f56c4c4b-c80c-418c-80d8-6a18f33b751a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:55.965861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:55.967724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:29:55.970215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b2565be-321b-4c0c-8613-583540419f14]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.970215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:29:55.970215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:29:55.970215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:29:55.970215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:29:55.972276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71936630-796a-4a20-b0ba-16442b35f86a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.974689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e427f8c-9e2f-44bf-b7f1-c843bb0d2530]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:55.976205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[420537a2-06f4-49d5-83d1-715d85296bf2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:56.006335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90aaf71c-b6b4-4673-bb0d-0d1bb0727ce9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 629198, 'reachable_time': 27876, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 198552, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf5abacc-c218-4522-af23-e4d7a08622cd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:29:56.012711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6deb3207-20b8-427e-bc26-09675ad7cd79", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.172", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "77850af0-a181-4d55-94e9-66b48525994d", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5244, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bf:7e:06", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "207c801c-3a6a-4381-9136-8c987e421fe4", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:29:49Z", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:29:49Z", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:29:53Z", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 42, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:29:56.020101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id cc07547b-2350-438c-a71d-8175a1d653e5. Wait time elapsed: 6.136 Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id cc07547b-2350-438c-a71d-8175a1d653e5. Wait time elapsed: 6.136 Jul 01 10:29:56.038591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:29:56.793002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.778s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:29:56.793616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:29:56.794339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:29:56.794749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:29:56.795093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:29:56.796335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7feeb8dd-4821-4439-bef4-8af42a4e5aa4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:56.810142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c522761f-de49-4302-b61d-a0cc88728279]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:56.832530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2beb3f30-534d-4866-b0ed-184351fa790f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:56.861648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcd31c0f-faa6-4cec-8c6f-5536b995781b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 198555, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:56.883282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0cf88c3d-fb1c-4883-97ed-fdce2817acea]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:56.903850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49fb4901-6c6b-4411-8f6d-0a6fa705ae53]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:56.924577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb12566c-ed60-407d-8f2d-4b241782bab5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:56.945309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2d49536-1593-41a2-bfed-3a10c9c75d43]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:56.945877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:29:56.946156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:29:56.946422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:29:56.946698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:29:56.962454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:29:56.962754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c0d106c-6beb-494f-afa6-5f50a5d2fab8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:56.965510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfcf7763-2e64-4712-b975-58e9a2c16e58]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:56.966500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:57.010894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:57.011707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:57.012194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:57.012650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:57.013119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:29:57.072653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4cce7ad-781c-4a32-b23d-c4aa421f3304]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:57.103507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ad31880f-67d4-4e17-aad2-ca48e1d91472]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:29:57.103507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:29:57.103507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:29:57.103507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:29:57.103507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:29:57.103507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:29:57.103507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:29:57.103507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:29:57.103507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:29:57.103507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:29:57.103507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:29:57.103507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:29:57.103507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad31880f-67d4-4e17-aad2-ca48e1d91472]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:57.104706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:57.106542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:29:57.109208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01339109-0aa8-4310-ac28-6c3e98291860]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:57.109208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:29:57.109208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:29:57.109208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:29:57.109564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:29:57.110431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[071b8c27-f122-40e1-adc4-a0035b8dfbdc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:57.111851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c82486c7-3e26-41b4-8d77-391c03688ceb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:57.112887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8edea72b-b7f2-4b7f-a6bb-6ff43b07a441]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:57.133925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54b258da-51f4-4b13-9648-99d349fea91a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 629312, 'reachable_time': 20920, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 198573, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[330e4d1a-1181-465d-b7e5-ca2da8b58eee]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:29:57.139464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6deb3207-20b8-427e-bc26-09675ad7cd79", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.172", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "77850af0-a181-4d55-94e9-66b48525994d", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5244, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bf:7e:06", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "207c801c-3a6a-4381-9136-8c987e421fe4", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:29:49Z", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:29:49Z", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:29:55Z", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 43, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:29:57.145109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id cc07547b-2350-438c-a71d-8175a1d653e5. Wait time elapsed: 7.260 Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id cc07547b-2350-438c-a71d-8175a1d653e5. Wait time elapsed: 7.260 Jul 01 10:29:57.160304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:30:00.125773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.987s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:30:00.126376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:30:00.126961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:30:00.127285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:30:00.127625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:30:00.128798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77e4020b-6173-4e6c-b56e-9adbe7f2bbff]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.157888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6af9601d-3836-4676-8f6a-9a937a3e9660]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.187994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65e1361a-f4e8-4cdb-a4b4-97c18011d4df]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.225408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9e6b4ec-f038-4408-b207-2414bd1d5333]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 198671, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.245925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03c44832-045e-4036-8962-6fd4fe886d20]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.267290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[369e6b77-33ff-4e0d-8d8a-60aa6319f42c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.288628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8cc85465-17bf-4e53-b21e-059860cda639]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.309253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ead98578-945c-4a89-b4a3-f5a3b7ba8781]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.309936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:30:00.310494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:30:00.310824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:30:00.311200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:30:00.327276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:30:00.327675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c63f3b5-defc-4a34-97fe-2870ef6db21c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.330880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bd3db64-b42d-4e55-8baa-525494fa8dc4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.331869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:00.366065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:00.367218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:00.367846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:00.368397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:00.368782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:00.419955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fd1dac3-372f-4d20-81ba-faf45a84a56d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.453475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b5374407-13ed-40c0-9601-f7bd45d26d81]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:30:00.453475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:30:00.453475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:30:00.453475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:30:00.453475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:30:00.453475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:30:00.453475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:30:00.453475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:30:00.453475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:30:00.453475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:30:00.453475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:30:00.453475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:30:00.453475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5374407-13ed-40c0-9601-f7bd45d26d81]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:00.454721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:00.456707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:30:00.459109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a05fc85-9b6b-417e-b72e-445e374e8e52]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.459109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:30:00.459109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:30:00.459109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:30:00.459109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:30:00.459914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54834f89-e6a9-436f-8b90-5326fcc0e4c3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.461113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63951572-ee3d-4347-b538-748a2a191abc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.462080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07150764-5ce8-4669-bb92-47234f17f50f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.481661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30c89377-6a92-4d91-ac95-8f51b0605874]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 629648, 'reachable_time': 30284, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 198723, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2274c1b-997e-4f23-b8db-7dced9fb9307]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:30:00.486197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6deb3207-20b8-427e-bc26-09675ad7cd79", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.172", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "77850af0-a181-4d55-94e9-66b48525994d", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5244, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bf:7e:06", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "207c801c-3a6a-4381-9136-8c987e421fe4", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:29:49Z", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:29:49Z", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:29:58Z", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 44, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:30:00.493497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id cc07547b-2350-438c-a71d-8175a1d653e5. Wait time elapsed: 10.608 Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id cc07547b-2350-438c-a71d-8175a1d653e5. Wait time elapsed: 10.609 Jul 01 10:30:00.508724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:30:01.294479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.806s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:30:01.295097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:30:01.298441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:30:01.298814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:30:01.299126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:30:01.300374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f853d64b-8c95-4ef9-88b7-c5ffe04ead7b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.314322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46c48c0b-5237-4ca7-bbc0-7e3f41dd356c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.336911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1fdf763-09d9-4c5b-9e7b-13b25ffc4c12]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.367770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36db1df4-a341-49ff-9624-a96fc168fa3c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 198726, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.388452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18feec98-32ce-4113-9ac7-d8bc66b8ea45]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.409869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8af4a28-2a96-4ea2-a188-7e71a1f05fcc]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.433134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[040313c9-4418-4c43-9a1d-349465570a56]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.456007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e56a0955-4506-4664-9bb4-17987ff98d0a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.456595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:30:01.457065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:30:01.457332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:30:01.457686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:30:01.472365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:30:01.472668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2371be4e-09ae-4a77-b305-9d588433a5a2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.476358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19d91f31-a87e-4006-b479-e18c52a690ac]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.477380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:01.530323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:01.530323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:01.531834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:01.531834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:01.531834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:01.604521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76c64713-87af-46fc-8786-946c7646ebe3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.646405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7cbbff63-91a0-43a2-aa01-ccbf6882cd0e]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:30:01.646405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:30:01.646405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:30:01.646405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:30:01.646405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:30:01.646405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:30:01.646405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:30:01.646405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:30:01.646405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:30:01.646405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:30:01.646405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:30:01.646405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:30:01.646405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:01.647613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7cbbff63-91a0-43a2-aa01-ccbf6882cd0e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:01.647884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:01.649839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:30:01.652619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f20b3194-ce8f-402a-b7b6-549939c07bad]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.652619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:30:01.652619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:30:01.652912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:30:01.653281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:30:01.654388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa7cfbed-dba1-46e8-abde-59a1cc06ae32]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.655376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6ea9fc6-691c-459f-b843-f8e06f3ce8a0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.656322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1809adba-54ab-49e0-a7ef-a400db0018b4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.676905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5a4fbca-6f0d-420d-a5e3-41ab8fbf9dcf]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 629763, 'reachable_time': 30156, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 198745, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.682479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:30:01.682740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02789af5-c1ad-41d2-aa15-683abe2a7dad]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:01.683924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6deb3207-20b8-427e-bc26-09675ad7cd79", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.172", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "77850af0-a181-4d55-94e9-66b48525994d", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5244, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bf:7e:06", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "207c801c-3a6a-4381-9136-8c987e421fe4", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:29:49Z", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:29:49Z", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:30:01.687010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:30:00Z", Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 45, Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id cc07547b-2350-438c-a71d-8175a1d653e5. Wait time elapsed: 11.807 Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id cc07547b-2350-438c-a71d-8175a1d653e5. Wait time elapsed: 11.807 Jul 01 10:30:01.704089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:30:04.517714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.831s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:30:04.518370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:30:04.518952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:30:04.519279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:30:04.519604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:30:04.521548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd71393b-3720-4023-b70b-d5484fcf743d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.539891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[551ab56c-44c5-414d-9a8c-23b1a4a1dbe8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.561048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12588541-2268-42ec-9845-28451da07a0d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.593812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[993129c2-aa83-4e6c-8596-47242acdc17a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 198751, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.615114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a69d32c1-99da-47e0-89d0-f91b226c29d4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.635503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3995176-edbd-46e6-9cfa-1abb71c8e150]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.657218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b41b53a3-011c-4422-ad1f-ae38e389f86b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.678421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0dfd58e-1253-4515-813d-e72b34ff25e3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.678994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:30:04.679331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:30:04.679649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:30:04.679920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:30:04.699752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:30:04.700129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e69735a-8af8-456d-9d8f-0b23e01f99d9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.703000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d153b52-a1ac-475a-8258-66e277f8cf26]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.704518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:04.749368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:04.750050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:04.750448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:04.750841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:04.751228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:30:04.798640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d183213-7064-42b5-bdcc-9504878bef45]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.834485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[94609f84-7ee9-494b-b825-1b6f4e674fe3]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:30:04.834485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:30:04.834485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:30:04.834485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:30:04.834485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:30:04.834485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:30:04.834485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:30:04.834485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:30:04.834485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:30:04.834485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:30:04.834485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:30:04.834485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:30:04.834485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94609f84-7ee9-494b-b825-1b6f4e674fe3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:04.835825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:04.838028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:30:04.840613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5845087-6552-448f-82b3-04094b2af5e9]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.840613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:30:04.840613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:30:04.840613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:30:04.840613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:30:04.842339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c505a5e7-8235-41f3-a2e1-e5ac6a904de8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.843497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68225e6a-9688-4bf5-a647-c47da241d1da]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.844406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[064f44ba-6eee-4750-a693-2675f5ec8d06]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.867741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aed19d0a-6577-41cc-82e2-b1b57b24ffd1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 630085, 'reachable_time': 31616, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 198769, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.869719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:30:04.870133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b79c4bf-c6e2-4e05-99b1-4f7b6b2e26b4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:30:04.871314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6deb3207-20b8-427e-bc26-09675ad7cd79", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.172", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "77850af0-a181-4d55-94e9-66b48525994d", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5244, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bf:7e:06", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "207c801c-3a6a-4381-9136-8c987e421fe4", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:29:49Z", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:29:49Z", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:30:04.876601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:30:01Z", Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 46, Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:30:04.889251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3 Jul 01 10:30:15.146062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:30:15.146740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:30:15.146740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:30:15.146740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:30:15.279372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:30:55.280803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:31:15.145990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:31:15.146613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:31:15.146613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:31:15.146840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:31:35.282603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:32:15.147099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:32:15.147850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:32:15.147850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:32:15.148057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:32:15.284565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:32:55.286111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:33:15.147829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:15.148508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:15.148508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:15.148630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:24.980985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-e6fc96d4-3de6-4988-8da9-1c2af07e43dc tempest-TestMinimumBasicScenario-2052601014 tempest-TestMinimumBasicScenario-2052601014-project] Got routers updated notification :['88bbc4fa-c162-4343-9551-ceded0fe88c3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:33:24.982060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id 9710578b-2a31-48e9-940e-64ee4c23fe9d. Wait time elapsed: 0.001 Jul 01 10:33:24.982364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id 9710578b-2a31-48e9-940e-64ee4c23fe9d. Wait time elapsed: 0.001 Jul 01 10:33:24.982957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:33:25.671609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.688s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:33:25.671883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:33:25.673310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:33:25.673671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:25.674366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:25.675810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45f5259f-2b5b-4e0a-a8f3-9d7c9a4719da]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:25.689398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9536d118-0c6d-4480-93be-d86f3fd45dc2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:25.711284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08932cc8-1f03-4a5b-a292-4250049ac0b1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:25.748666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11991aff-44d9-4595-94cf-5ebd82a44cb7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202132, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:25.770550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc9ad493-0f13-413f-8058-c3efda98ce6d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:25.792256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[656b4fa5-b75b-43f1-8de1-9d7e416eb17d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:25.819953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b517cc3-1716-429b-9b49-b6069d0d2462]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:25.848246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5460f6f-31ab-4cb3-8c8c-20f835f183e1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:25.849483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:25.850298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:25.850588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:33:25.850864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:33:25.871307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:33:25.871950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5748a0e-2d36-4fd5-a8ae-e61c905c41c6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:25.875906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0e58164-0bf1-4bc1-b99d-ef2a868acbdb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:25.877534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:25.926435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:25.927022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:25.932068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:25.932630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:25.933182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:25.994952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34996b34-9b56-4cb5-91ed-030e5002a8d2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:26.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[dea2a37f-c2db-415d-b013-beb4bc6b8797]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:33:26.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:33:26.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:33:26.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:33:26.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:33:26.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:33:26.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:33:26.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:33:26.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:33:26.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:33:26.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:33:26.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:33:26.034823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:26.036060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dea2a37f-c2db-415d-b013-beb4bc6b8797]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:26.037142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:26.039615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:26.043395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[070c1d1e-587d-40f6-a1c8-ac672872ee10]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:26.044317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:26.044641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:26.044895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:33:26.046417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:26.047507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0da813c5-820b-4aac-a4a7-d02016752d1a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:26.049470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a902c2c-aa27-4b47-ae3c-5f18ea5b0010]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:26.050584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c4ddc74-8620-496e-97db-b6352d350200]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:26.073443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[280dd204-5efc-4022-8e21-6d626b3376f5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 650201, 'reachable_time': 20116, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202153, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:26.083650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:33:26.084045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cf7bcf8-2808-4603-9d66-925da063708f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:26.085693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:30:07Z", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 51, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:33:26.087906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 9710578b-2a31-48e9-940e-64ee4c23fe9d. Wait time elapsed: 1.106 Jul 01 10:33:26.109507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 9710578b-2a31-48e9-940e-64ee4c23fe9d. Wait time elapsed: 1.106 Jul 01 10:33:26.109507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:33:29.300573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.212s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:33:29.301515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:33:29.301926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:33:29.302305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:29.302529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:29.304356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5332be6f-432c-45cc-b17e-f29c3a3580b1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.331275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a425c583-ec4e-4171-b49c-4fb9714fc235]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.353626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4486c2b4-19cd-4b1a-bf2f-ad77ce05112e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.383762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[698c462c-fc53-4658-9cca-487d75d97e1f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202241, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.412521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb3aee4a-1a33-448c-990e-175d000f85a2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.444458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93ea7ed7-68d7-44d4-ad28-145b870d81b8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.468424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e55d26a3-09e3-498f-bbc2-79c86206fdd7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.495423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bfef895-e6e1-4863-9d42-dea8467c9208]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.496352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:29.497146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:29.497478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:33:29.497820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:33:29.518747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:33:29.519242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88477ed8-51eb-4e95-bcf0-4236646bbfe2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.522376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03d439e9-465b-4d77-8375-236380e0572a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.523758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:29.557995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:29.559137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:29.559708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:29.560168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:29.561907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:29.634758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[916fabe8-f757-4eb3-a450-48efce6a34c3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.681831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3c4831d1-188a-4173-8ef4-3063e5443639]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:33:29.681831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:33:29.681831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:33:29.681831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:33:29.681831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:33:29.681831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:33:29.681831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:33:29.681831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:33:29.681831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:33:29.681831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:33:29.681831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:33:29.681831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:33:29.681831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:29.682747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c4831d1-188a-4173-8ef4-3063e5443639]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:29.683029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:29.684706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:29.686791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48cbb062-d67f-474e-b3a2-84c624dd3e54]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.686791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:29.686791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:29.686791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:33:29.686791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:29.687639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75d11593-5067-4c2f-8119-16d4065ad162]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.688660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f328069-9432-4c47-81c5-1527ae2acac0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.690660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8a4ae78-65d3-4c2a-a76e-311291f6b589]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.717928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab56e1b2-fa33-4f00-88e4-3605a60cbb0a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 650564, 'reachable_time': 23960, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202261, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.728898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:33:29.729316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[554238c8-2d30-4c94-bb33-71c7bd6a38d1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:29.730511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:30:07Z", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 51, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:33:29.734535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:33:29.748638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 9710578b-2a31-48e9-940e-64ee4c23fe9d. Wait time elapsed: 4.764 Jul 01 10:33:29.748638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 9710578b-2a31-48e9-940e-64ee4c23fe9d. Wait time elapsed: 4.764 Jul 01 10:33:29.748638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:33:30.560280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.814s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:33:30.560838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:33:30.561734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:33:30.562418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:30.562846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:30.564926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29cff051-d35f-4877-8c75-54383de36ce9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.581796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d025d25-f623-48da-b8fe-35710640250d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.604059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0925b2e5-e539-400e-89a6-1a901a721003]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.643342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[735dedf9-001b-431c-b45e-5541359a34b1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202267, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.682288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26dcee73-ec9a-475b-9ec1-23bab00740d5]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.706136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9638012-ed3c-4481-9858-032306aefd59]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.742903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c781e7c-45ad-40dc-a25e-3d4c4cc76c0f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.771014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87f54613-ebc7-44e5-9e9c-17245f92c844]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.771684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:30.771953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:30.772363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:33:30.773947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:33:30.794364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:33:30.795221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e46252a-ca6a-470d-a318-7ff873b8a227]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.799198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf201452-29ad-4682-afde-03149f6e0713]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.800554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:30.846682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:30.847686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:30.848226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:30.848839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:30.849784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:30.928216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f75fc74-653d-4fdf-9015-a404eeccfa54]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.971483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[60234e05-1538-4c7d-a627-a5ccaecadab3]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:33:30.971483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:33:30.971483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:33:30.971483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:33:30.971483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:33:30.971483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:33:30.971483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:33:30.971483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:33:30.971483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:33:30.971483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:33:30.971483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:33:30.971483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:33:30.971483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:30.972494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60234e05-1538-4c7d-a627-a5ccaecadab3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:30.972710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:30.974044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:30.975732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89329ffe-2a8a-4157-8eb8-1bbc1a75dadb]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.975732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:30.975933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:30.976170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:33:30.976644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:30.977799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dae4c147-24de-4a54-8b96-16912e1dd40a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.978966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[084774ca-52be-4cdd-881f-cd35bf608a0b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:30.980162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[782e2191-a41a-4803-b19f-b328c2219204]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:31.000856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51b39760-c05f-4f43-9ce6-80782db722d0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 650690, 'reachable_time': 36512, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202288, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:31.007894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:33:31.008309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2fcbce1-aee4-4b48-b771-a7bdb7c009cd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:31.009420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:28Z", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 52, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:33:31.011589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 9710578b-2a31-48e9-940e-64ee4c23fe9d. Wait time elapsed: 6.030 Jul 01 10:33:31.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 9710578b-2a31-48e9-940e-64ee4c23fe9d. Wait time elapsed: 6.030 Jul 01 10:33:31.024638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:33:34.358942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.347s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:33:34.360121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:33:34.361384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:33:34.361950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:34.362605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:34.364035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6688f71d-227f-4f1f-980a-561d5462a633]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.402943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e483bbd-2be1-4173-a286-22522b0be5d2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.502613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[befde6b1-fae1-4b58-b4e2-de1e72b9870a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.551443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19301e1a-3af9-4b96-af09-6a1282d4d122]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202368, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.583617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40d4c959-e0a8-4191-ac21-49cba46e91c3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.612799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11806302-ec22-4034-8b03-12d294889ea5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.639290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de65b1b3-0bb0-4cb8-9fa2-62572834c63f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.668693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75c96c69-7a91-4585-9ab0-cbe54f8c9d51]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.669791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:34.670068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:34.670333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:33:34.670615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:33:34.691239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:33:34.691675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad06c5ab-ccd6-47dc-8f54-dd20098a4765]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.694501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0625eb53-a01d-4532-935f-b8eb89174948]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.695546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:34.742531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:34.743735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:34.744308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:34.744839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:34.745694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:34.787004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5cdb5db-fb7a-4af6-8c0b-0f42bce1aed1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.830615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3666d37f-4ae8-4a96-8a68-dc4836e465bc]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:33:34.830615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:33:34.830615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:33:34.830615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:33:34.830615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:33:34.830615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:33:34.830615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:33:34.830615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:33:34.830615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:33:34.830615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:33:34.830615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:33:34.830615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:33:34.830615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:34.831458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3666d37f-4ae8-4a96-8a68-dc4836e465bc]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:34.831990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:34.835131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:34.839250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2e3fb29-072a-43f6-bccc-360d2ba4504b]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.840108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:34.840428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:34.840681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:33:34.841018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:34.841971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eea9e801-e434-4bd6-b72f-29595df74977]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.843903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6323e51e-ccaf-4ed7-875b-df0fcbb339e2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.844792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[419657a3-9655-48e9-b589-b7f75f1d0cf5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.873867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c38b448-9765-4895-88dd-d652f93e2860]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 651081, 'reachable_time': 28836, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202392, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.881479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:33:34.881835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9cc326d-f34f-4bea-ab49-c742dea35a14]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:34.882993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:32Z", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 55, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:33:34.886334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:33:34.894986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 9710578b-2a31-48e9-940e-64ee4c23fe9d. Wait time elapsed: 9.908 Jul 01 10:33:34.895272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 9710578b-2a31-48e9-940e-64ee4c23fe9d. Wait time elapsed: 9.913 Jul 01 10:33:34.895804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:33:35.287173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:33:35.601415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.706s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:33:35.602009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:33:35.602595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:33:35.602938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:35.603215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:35.604587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f3a1b1d-8adc-421a-823d-fe60cb02c9d9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.617000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09dde80e-fc66-48b0-8693-4d5a10a1bc30]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.638972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef0d2157-2bf8-4c20-806e-5de5db0c087b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.666020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ceadb5f-0515-4d25-ac2f-777690f39069]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202396, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.687626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd391bf9-9feb-4fc6-a417-1e06b5cc423b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.713242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[521d5783-542f-4506-bb7a-a13e5719f5ac]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.734419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[623fcc67-7117-4f11-897f-c40f764a1c38]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.756497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c968c6b-bd61-400e-9abe-2c5620686062]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.757048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:35.757325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:35.757620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:33:35.757901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:33:35.779973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:33:35.780270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8e85238-1788-4064-ae1c-bc1d4e9fd69b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.783062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6d52202-155f-47f1-b45f-81a869b64416]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.784292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:35.832466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:35.834678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:35.836618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:35.837346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:35.839771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:35.908493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[816d50d5-5f2e-4605-a8e0-6b110aa0b68c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.947418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7e698aac-83cb-4500-a1e4-148c5adec034]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:33:35.947418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:33:35.947418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:33:35.947418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:33:35.947418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:33:35.947418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:33:35.947418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:33:35.947418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:33:35.947418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:33:35.947418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:33:35.947418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:33:35.947418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:33:35.947418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:35.948216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e698aac-83cb-4500-a1e4-148c5adec034]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:35.948698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:35.949999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:35.951739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[852e3d43-0c09-4716-a7b0-297efab9a0f7]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.951739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:35.952120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:35.952389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:33:35.952733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:35.953761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df86943a-0679-46de-9bf5-383e3d549e68]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.954815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4970848e-c151-42e6-bab7-46083850de99]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.956352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f532adff-53c4-4e77-bd64-f5297af59bbb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.975545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fed64f0-383a-4fb8-b018-a8f031dcabd5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 651192, 'reachable_time': 17844, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202415, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b82a6a70-9d37-4982-8ffe-1e2c92cdada6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:33:35.980589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:34Z", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 57, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 9710578b-2a31-48e9-940e-64ee4c23fe9d. Wait time elapsed: 10.999 Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 2, update_id 9710578b-2a31-48e9-940e-64ee4c23fe9d. Wait time elapsed: 10.999 Jul 01 10:33:35.984464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:33:38.485823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.505s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:33:38.486469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:33:38.487502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:33:38.487665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:38.488193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:38.489787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[040fe112-36f2-4ec4-840d-7a3159c409cb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.506306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05d37f78-0b68-42a2-89a4-7c7af93a95bf]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.530485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7794eddd-6291-4f18-82fa-fbde1446f142]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.563917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa2d9ada-6236-4c18-a6ca-efe19761d3d5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202423, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.589655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[488f9433-9d38-4b0b-b211-2d3add7c876a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.613852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[290028d7-5911-46b4-9816-3077478b4593]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.637337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5d0d1d9-4454-4a74-8302-e5a2e9b35d8b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.662368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50a36218-6926-4771-bd3a-19270444dcd2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.663135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:38.663709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:38.664116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:33:38.664469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d00573fe-0be0-4873-9cbd-0476472520a1) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:33:38.681657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:33:38.682033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38d9d6e4-77af-4068-9536-2fd3eac44239]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.684774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf950aab-78b1-4819-946b-03a176a2f296]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.685875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d00573fe-0b, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:38.720376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d00573fe-0b, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:38.722365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:38.722926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:38.723573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d00573fe-0b, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:38.724368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d00573fe-0b, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd00573fe-0be0-4873-9cbd-0476472520a1', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a9:38:7b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:38.797213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b35793c3-209d-4811-acdd-2a53e15c7a68]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.845930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[81a08a7b-ec54-422c-9d92-c22b86f4a6fe]: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:33:38.845930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:33:38.845930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:33:38.845930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:33:38.845930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:33:38.845930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:33:38.845930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:33:38.845930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:33:38.845930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:33:38.845930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:33:38.845930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:33:38.845930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:33:38.845930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:38.846873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81a08a7b-ec54-422c-9d92-c22b86f4a6fe]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:38.847691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:38.849373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:38.851417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10aa0972-6fd8-4a96-8ae4-9dc5cf6ffc72]: (4, ['qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.851417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:38.851417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:38.851417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:33:38.851417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:38.852551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9801b835-e1f1-40c6-ba95-06d23c23c188]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.853786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[beb8c387-244b-4751-8555-33740a496177]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.854712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[421989a0-6eb2-4dfe-a623-3fd0d1eb7581]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.884384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e9c894a-8265-4d1f-8198-023b6f7eb99f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 651482, 'reachable_time': 37024, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202441, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.888822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:33:38.889374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58827826-132b-4723-b68a-62155db56693]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 88bbc4fa-c162-4343-9551-ceded0fe88c3: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d00573fe-0b not found in namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3. Jul 01 10:33:38.890221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestMinimumBasicScenario-1838720146-router", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249" Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:04Z", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3079fda2-a673-4bf2-ba86-4126326cd8c8", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:9e:70:39", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4910, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.249", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:01Z", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:24:06Z", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d00573fe-0be0-4873-9cbd-0476472520a1", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "ebb1c2a9-f7ff-4903-ab4f-5784b66e2708", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a9:38:7b", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "88bbc4fa-c162-4343-9551-ceded0fe88c3", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 4912, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:24:03Z", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:35Z", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 58, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "fad26a39bbb6455baa4d49af9909940d", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "364f0f40-fb26-4171-a7ea-1093c4860f87", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:33:38.894559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:33:38.905537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3 Jul 01 10:33:40.608734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-a8903282-d6db-4ee4-afdd-3b5d6eabef73 admin admin] Got routers updated notification :['88bbc4fa-c162-4343-9551-ceded0fe88c3'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:33:40.608734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id 8090d67e-b6bd-464e-8168-fd3539fed701. Wait time elapsed: 0.000 Jul 01 10:33:40.608734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 3, priority 1, update_id 8090d67e-b6bd-464e-8168-fd3539fed701. Wait time elapsed: 0.001 Jul 01 10:33:40.608734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:33:41.247476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.639s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:33:41.248407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 88bbc4fa-c162-4343-9551-ceded0fe88c3 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:33:41.251093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:33:41.252434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:41.253414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:41.255742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b184abd-d696-4219-ac5d-97480295cd31]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.271481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7140eae-4797-4e10-9bea-8fd2ba9723a3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.292289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf7e2dad-6810-4d28-98b9-bb54d99afde0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.325868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22965051-583b-4ae3-8df9-f6dcdde4aa29]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202506, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.348468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d87eb19c-76ae-4af5-b682-7ff21103055c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.371310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3dfc571c-db32-46bf-9dc0-83209ef1ba75]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.393293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8767af6d-f5f8-491a-8f5d-19b845705c03]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.415401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51c34ba6-e6b7-4da4-be3b-c84f731e69fc]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.416643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:41.417061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:41.417473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:33:41.449860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ec342e5-a7bd-42bb-b0e1-ca3caa97d651]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 651758, 'reachable_time': 20112, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202512, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.450650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:41.451075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:41.451715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:33:41.451976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:33:41.469830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-bff5a72f-f47b-4aa0-8500-6ae10970d33d admin admin] Got router deleted notification for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:33:41.476489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b050a707-78ea-4924-b2da-964d08f422b1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 651758, 'reachable_time': 20112, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202513, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.477768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:33:41.478444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:33:41.492343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36d16687-5d34-417c-ab26-a4a8dfa460e6]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.691711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[629ad187-61be-418b-b75c-22bbbbbb1b55]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.717658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d96d6c4b-e21f-46a7-9acc-c75db8ed8a77]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.898469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[372c301e-7722-47d7-bf50-d8930be86f76]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.899397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:33:41.899716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:33:41.899996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:33:41.900270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" released by "process_external" :: held 0.448s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:33:41.900883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:33:41.901133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:33:41.901550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:33:41.901990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:33:41.929515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f339b358-7607-4f6e-b19d-95fc69e200fb]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 10:33:41 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:33:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:33:41 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 10:33:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:33:41 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 10:33:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:33:41 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 10:33:41 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.970645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49a204e8-8668-4790-8d80-352bcd152c34]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:41.992337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22d5f898-4f58-4194-a4fd-7a14ea149764]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:33:41 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:33:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:33:41 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 10:33:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:33:41 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 10:33:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:33:41 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 10:33:41 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.024603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31422e67-51e2-4724-8ad5-f69ddd42789a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.025093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:33:42.025416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:33:42.025673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:33:42.025951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" released by "process_address_scope" :: held 0.125s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:33:42.026314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:33:42.026764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:33:42.027009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:33:42.027386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:33:42.027832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:33:42.042850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0d3f0d9-4c4a-4e28-9995-eca5c07bdcde]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 10:33:42 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:33:42 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:33:42 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 10:33:42 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:33:42 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 10:33:42 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:33:42 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 10:33:42 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.102464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9feb9ed9-60bf-4f5c-bc0e-6d5e71072a58]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.127453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b5b7e4d-e419-481d-b255-fa4bc8fc1fc1]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:33:42 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:33:42 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:33:42 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 10:33:42 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:33:42 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 10:33:42 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:33:42 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 10:33:42 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.174952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ea53851-371b-428d-a09a-39d19a799a46]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.175465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:33:42.176354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:33:42.176354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:33:42.176354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3" released by "apply_metadata_nat_rules" :: held 0.149s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:33:42.176802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-88bbc4fa-c162-4343-9551-ceded0fe88c3 Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.haproxy Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 88bbc4fa-c162-4343-9551-ceded0fe88c3 Jul 01 10:33:42.177235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 10:33:42.179491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/88bbc4fa-c162-4343-9551-ceded0fe88c3.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 10:33:42.206186 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-88bbc4fa-c162-4343-9551-ceded0fe88c3[202529]: Proxy listener started. Jul 01 10:33:42.211563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 88bbc4fa-c162-4343-9551-ceded0fe88c3 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 10:33:42.211834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, update_id 8090d67e-b6bd-464e-8168-fd3539fed701. Time elapsed: 1.605 Jul 01 10:33:42.212100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 1, priority 1, update_id 3b4b9008-2968-4078-9c28-c578535ecf36. Wait time elapsed: 0.742 Jul 01 10:33:42.212340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 88bbc4fa-c162-4343-9551-ceded0fe88c3, action 1, priority 1, update_id 3b4b9008-2968-4078-9c28-c578535ecf36. Wait time elapsed: 0.742 Jul 01 10:33:42.212625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 88bbc4fa-c162-4343-9551-ceded0fe88c3 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 10:33:42.213167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:33:42.233589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[226c3e5f-f6ad-4856-af4b-a1ba0b93a415]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.237478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9df5ae13-0eb6-49ed-9881-63fb8106628c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.239633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f642c92-e4b3-4ebe-adde-23f6198beb77]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.240277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 10:33:42.241494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51768ac8-6832-41e3-94e1-cdf9b89687ad]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.269779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[796e7ec1-255e-4341-bf34-2e91aa0b37a5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 651758, 'reachable_time': 20112, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202535, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.270972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:42.271469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:42.301160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8fb57ca-cd7c-4866-98b4-4bc4109d6f9f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 651758, 'reachable_time': 20112, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202536, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.302262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 10:33:42.302740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:42.303086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/88bbc4fa-c162-4343-9551-ceded0fe88c3.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:42.303341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:33:42.304121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 88bbc4fa-c162-4343-9551-ceded0fe88c3 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 10:33:42.305170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf363ed5-4cdb-4d02-b77d-aa876c1a5ea7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.352133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[395d8a4f-bc8b-443e-b659-8d137b4e02cf]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 651758, 'reachable_time': 20112, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202537, 'error': None, 'target': 'qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.358411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-88bbc4fa-c162-4343-9551-ceded0fe88c3 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:33:42.358411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d02ee6f2-32f4-4a7f-91cb-ccfcd207ed5a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:42.359131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 88bbc4fa-c162-4343-9551-ceded0fe88c3 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 10:33:42.359574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (88bbc4fa-c162-4343-9551-ceded0fe88c3), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:33:42.360279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:42.360644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:42.361496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:42.362222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:42.362478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 88bbc4fa-c162-4343-9551-ceded0fe88c3, update_id 3b4b9008-2968-4078-9c28-c578535ecf36. Time elapsed: 0.150 Jul 01 10:33:43.996287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-2edc1312-7fef-4092-9c12-d15d0421014d admin admin] Got routers updated notification :['c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:33:43.998780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 1, update_id 2b3995d5-f242-431e-a1e9-0c7fc400900d. Wait time elapsed: 0.001 Jul 01 10:33:43.999353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 3, priority 1, update_id 2b3995d5-f242-431e-a1e9-0c7fc400900d. Wait time elapsed: 0.001 Jul 01 10:33:43.999813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:33:44.968021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-7497a1c4-ad1c-4dba-a316-723578cee5d9 admin admin] Got router deleted notification for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:33:45.008777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.009s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:33:45.009616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 was not found. Performing router cleanup Jul 01 10:33:45.011269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75d3918d-1f61-4806-b807-2b6ea18f79e6]: (4, ['qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:45.012219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:45.012496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:45.013399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, update_id 2b3995d5-f242-431e-a1e9-0c7fc400900d. Time elapsed: 1.014 Jul 01 10:33:45.013679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 1, priority 1, update_id c232bc9b-a871-40a7-9f16-562279436c18. Wait time elapsed: 0.045 Jul 01 10:33:45.013922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, action 1, priority 1, update_id c232bc9b-a871-40a7-9f16-562279436c18. Wait time elapsed: 0.045 Jul 01 10:33:45.014172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2 was not found. Performing router cleanup Jul 01 10:33:45.015737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e661f55-7532-43bd-9642-2e0b7c792827]: (4, ['qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:45.016296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:45.016571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:45.017169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for c0e0e8e9-e7e4-417d-b641-ac8335ebbdd2, update_id c232bc9b-a871-40a7-9f16-562279436c18. Time elapsed: 0.003 Jul 01 10:33:52.424192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ba28997d-b62b-4421-ad8d-2bc86fca509e tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['737640e8-679f-4286-82e5-4dbeca9e7635'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:33:52.426026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 1, update_id 3aa2e6eb-8c50-454a-9acd-45009aaca73e. Wait time elapsed: 0.001 Jul 01 10:33:52.426675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 1, update_id 3aa2e6eb-8c50-454a-9acd-45009aaca73e. Wait time elapsed: 0.001 Jul 01 10:33:52.427425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:33:53.738556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.311s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:33:53.739717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:33:53.741426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:33:53.742356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:53.742857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:53.744619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3740f1c1-63ac-4a48-8bee-42c368416dc8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:53.770963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3df2d71-22b8-40bb-a19d-3e9a8e655574]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:53.794013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6aaad15e-a859-4d83-b73c-15239d46d09c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:53.828571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41516688-d42f-4f6c-99f8-24ea328446e1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202687, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:53.856517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[886bad2d-3933-4c38-b6c2-7ec0819dcf53]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:53.884558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12f5af46-ba5c-49d0-81b5-58aa97746d58]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:53.911991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b79fe9f4-fbda-4688-9dca-b4874cd7fdcc]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:53.933566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7200341a-ebf3-4a26-8ce3-4a7ec14b3151]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:53.934230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:53.934570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:53.934907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:33:53.959735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ccef5c3e-7ae7-4ded-9283-2806556bbeb8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 653009, 'reachable_time': 25692, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202693, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:53.960965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:53.961262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:53.961709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:33:53.961953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:33:53.962286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:53.962563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:53.963345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '8a3cff40-2ad5-4ecd-8572-4922c3d45db4', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:16:1c:f6', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '737640e8-679f-4286-82e5-4dbeca9e7635', 'device_owner': 'network:router_gateway', 'standard_attr_id': 5500, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.101', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:33:51Z', 'updated_at': '2022-07-01T10:33:53Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-8a3cff40-2a), ns(qrouter-737640e8-679f-4286-82e5-4dbeca9e7635) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:33:53.982149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:33:53.982615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e582a6a-6b5a-493e-9a20-4f14ce4c5756]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:53.988145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82871f0c-7781-458f-b1d5-aecb73e1771b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:53.989368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-8a3cff40-2a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:53.989767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:33:53.990574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-8a3cff40-2a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:53.991452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-8a3cff40-2a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:53.991849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-8a3cff40-2a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:53.992237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-8a3cff40-2a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:53.992622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-8a3cff40-2a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8a3cff40-2ad5-4ecd-8572-4922c3d45db4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:16:1c:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:54.053547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c4cf29b-da45-4155-91fb-ab64bfde58a0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:54.097535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cbd0c632-4494-4a0e-8c03-bef7743d1300]: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:33:54.097535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:33:54.097535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:33:54.097535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:33:54.097535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:33:54.097535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:33:54.097535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:33:54.097535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:33:54.097535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:33:54.097535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:33:54.097535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:33:54.097535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:33:54.097535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:54.099893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbd0c632-4494-4a0e-8c03-bef7743d1300]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:54.101050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:33:54.101880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:33:54.136279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcfb1829-7a74-4a7d-910f-19161d518f32]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:54.331244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b77c3410-3a98-4504-abcd-de60ae62dce3]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:54.367023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ddb992c-6b0b-4076-9564-9ad1e22822a3]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:54.555075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f269f73-a6b9-463c-b357-21d4422914e2]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:54.555618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:33:54.555938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:33:54.556234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:33:54.556526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" released by "process_external" :: held 0.595s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:54.557587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:54.559379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:54.561508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a738697d-f55a-4a16-b9f2-fb21c54a3963]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:54.561508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:54.561508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:54.561508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:33:54.561869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:54.562757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ea279fa-06e0-493c-a300-63eefda9c1e1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:54.564558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4956e122-186b-494a-9034-f5f03f904117]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:54.566596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d08a37f-6d99-4798-82fd-80c063cebbaa]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:54.585941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89ccfff4-3c78-45fc-b31c-8902b38720a3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 653009, 'reachable_time': 25692, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202740, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:54.593014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:33:54.593014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d1d2e45-f576-4c45-8509-cc9a9a71525a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:54.593983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:51Z", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:53Z", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:33:54.599504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:33:54.605728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 3aa2e6eb-8c50-454a-9acd-45009aaca73e. Wait time elapsed: 2.175 Jul 01 10:33:54.605728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 3aa2e6eb-8c50-454a-9acd-45009aaca73e. Wait time elapsed: 2.175 Jul 01 10:33:54.605728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:33:55.350214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-5c1cd401-13fd-4e31-a834-c51f9160d6ea tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['737640e8-679f-4286-82e5-4dbeca9e7635'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:33:58.859912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.258s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:33:58.863101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:33:58.864737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:33:58.867096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:58.870856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.004s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:58.875097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d18c9a6-2305-44bf-9f5f-44e9d3a6e7bc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:58.914041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0dcf8e8-41f4-40b3-ad21-e4867dd71d1a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:58.972154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8df0e9b2-bc5f-43d8-bc49-43ed2fd58c50]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.024035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b454126d-f87b-4556-9711-6e80bb1a452c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202763, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.078714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1027bc42-6e1c-4bce-a491-542eda50382f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.120911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7829d17-4391-4e42-a8c1-f6c64941ab83]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.165907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0e400bb-0dee-40cc-b38e-b49f2e72474b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.192622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a9994ca-09e7-43eb-9ee6-8f8711776cdc]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.194634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:33:59.195525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:33:59.195761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:33:59.196136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:33:59.230162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:33:59.230841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a2ec5be-1597-4be1-adab-a9dd9cf0fb68]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.234401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b9fa9af-b822-4e2f-a5ae-610bd7ac9d01]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.236636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:59.237241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:33:59.238253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:59.240134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:59.240901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:59.243681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:59.244310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:33:59.293916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a0d18b9-da47-460c-a9bf-0c7e41fe2d2c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.365107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8cd5dc68-c881-435d-98c2-473b33ceedbe]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:33:59.365107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:33:59.365107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:33:59.365107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:33:59.365107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:33:59.365107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:33:59.365107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:33:59.365107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:33:59.365107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:33:59.365107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:33:59.365107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:33:59.365107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:33:59.365107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:59.366240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8cd5dc68-c881-435d-98c2-473b33ceedbe]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:59.367409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:59.370036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:59.372530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f1d44d8-61df-407b-962b-5842eb67900c]: (4, ['qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.373505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:59.373932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:59.374257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:33:59.374691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:33:59.375871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2313d619-0e89-46ff-a5ac-e0a4ea6152b7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.376981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b90eb68e-dda7-4990-9024-809e776b8306]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.377971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3e442b2-f4a5-4aca-abb7-9d1764964915]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.416453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5de835ae-9dcd-4f78-a61e-84b0fb611796]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 653528, 'reachable_time': 30164, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202812, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.440033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:33:59.440518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2cb8638a-cb61-4ac4-a6ff-ff8cd293a9b7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:33:59.441937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:57Z", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:33:59.445114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:33:59.469365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 1, update_id 71f177e9-6b11-42c3-a122-7bbd9049bae6. Wait time elapsed: 4.112 Jul 01 10:33:59.469365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 1, update_id 71f177e9-6b11-42c3-a122-7bbd9049bae6. Wait time elapsed: 4.112 Jul 01 10:33:59.469365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:00.262272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.799s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:00.262889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:00.263714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:00.264095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:00.264446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:00.265689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c7c06a5-ee58-4096-87ae-0f6eef916e39]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.278798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef94f667-dea3-4b75-8951-fde1ebfcd82c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.300800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3705bf4-9fe7-44b0-acaf-51d056d0bff1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.332999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2730b2a-db12-42be-a911-ca32ddf6ca6c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202829, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.354075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67b5cda7-a2d2-42a6-b1d3-34d715243363]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.375117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31d19a61-0e39-4146-8d31-b5113ef93c3a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.396041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34a70a3f-20c1-4b20-a6a0-3829b5f5dab0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.416613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddf1d5b5-8cb9-4335-86e1-387f68472cfa]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.417172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:00.417609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:00.417884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:00.418164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:00.436983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:00.437382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c33c4749-01ab-4ddd-a6b0-2f4146e79bd0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.441435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9658774a-baab-4a16-b1a3-7f9096092212]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.442535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:00.478937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:00.479638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:00.480034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:00.480416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:00.480796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:00.530256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c756fec-5514-4e48-a420-aadf88516a15]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.562187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[335b96c8-ffdd-45ec-89cc-8088792759a4]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:00.562187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:00.562187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:00.562187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:00.562187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:00.562187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:00.562187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:00.562187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:00.562187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:00.562187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:00.562187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:00.562187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:00.562187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:00.564226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[335b96c8-ffdd-45ec-89cc-8088792759a4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:00.565427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:00.567113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:00.571173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbe7a983-ccab-4e71-911d-8db146725680]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.571958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:00.572275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:00.572526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:00.572856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:00.574539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc1e47b7-be1e-4a28-88c0-78c97fc979b6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.575512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6accdb68-3c03-46ee-ab23-6ccd0bb872ab]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.576492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9fef211-6584-4f9d-b9ff-49161a68040a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.595950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ea2e708-f171-48ac-8511-dff821f3b65d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 653659, 'reachable_time': 23124, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202847, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.598576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:00.598998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e083e0ff-42d7-400c-9337-13a020190238]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:00.603112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:57Z", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:00.605563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 3aa2e6eb-8c50-454a-9acd-45009aaca73e. Wait time elapsed: 8.179 Jul 01 10:34:00.617745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 3aa2e6eb-8c50-454a-9acd-45009aaca73e. Wait time elapsed: 8.179 Jul 01 10:34:00.617745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:02.624433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.019s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:02.625040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:02.625761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:02.626092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:02.626650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:02.627584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b2429e3-7803-49e8-8631-52bde09f68f2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.642373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79aee7b8-9691-4414-a2e4-776258a6c0b7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.663564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e3d5729-e5aa-4d96-9876-9a4627069b63]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.692251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2bdddf47-2ef5-4943-8195-206e83903f53]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202850, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.714361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34059c49-a55b-4702-90b8-26cd65775d37]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.736473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be9421cf-a8ae-4a60-b1e7-fd7061619b2e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.757513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b17f3c9-1b2b-456a-92bc-4192976793e9]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.778995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79cfc4b8-9852-4977-8d61-c04893c7be57]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.779859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:02.780026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:02.780292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:02.780569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:02.796266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:02.796591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b703bbfd-8ce3-4d87-a6a5-abadf8a1ce83]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.799916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5380e733-7f50-4af6-90aa-176da1ebec77]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.800881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:02.839720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:02.840497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:02.841033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:02.841618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:02.842201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:02.895722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db4a5ea9-d3ea-4917-86a7-2bb63486969d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.930587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[947199e4-0a34-4b75-ac0c-cb562326fd87]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:02.930587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:02.930587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:02.930587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:02.930587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:02.930587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:02.930587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:02.930587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:02.930587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:02.930587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:02.930587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:02.930587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:02.930587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:02.931621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[947199e4-0a34-4b75-ac0c-cb562326fd87]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:02.931955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:02.933316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:02.935052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58787125-e6e8-4046-97b7-0af4e2ba026e]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.935052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:02.935052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:02.935285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:02.935648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:02.936684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58bf0727-d309-4b3d-9749-a137a7bbde0d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.938066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[609d7b1b-9efe-4d2a-8336-1ee495005498]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.939537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a783f2e-903c-48f9-b904-402781dbc45b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.975941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c58e444-a578-48b3-acce-3ef86a9877d5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 653895, 'reachable_time': 39952, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202869, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.982175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:02.982466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eca47a15-d241-4bb8-ad1c-32fd41df31e7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:02.983539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:00Z", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:02.986762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 71f177e9-6b11-42c3-a122-7bbd9049bae6. Wait time elapsed: 7.634 Jul 01 10:34:03.006534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 71f177e9-6b11-42c3-a122-7bbd9049bae6. Wait time elapsed: 7.634 Jul 01 10:34:03.006534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:03.769581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.784s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:03.770181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:03.771639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:03.772048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:03.772407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:03.774040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a272a73-3dc6-4efa-8a5b-4ce0bc014c9c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:03.789639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[956f2576-9891-4c5a-98c0-016f87a40a6f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:03.812053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcb5431d-ce07-4c2d-ab3f-c923d81c8d76]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:03.839568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[870433c2-9bf6-4000-b1e1-e32954c2e80c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202873, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:03.861259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[877fe2f8-9317-4c85-a133-5d681a044298]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:03.883090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2d1f4f3-1998-4df8-b9cd-55e382869ace]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:03.907145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[085aade4-b05d-438e-9d63-d0d283fa5f0d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:03.933018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d339d62f-5017-4de6-ae67-3ddae4f0d1b4]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:03.933674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:03.933990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:03.934556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:03.935026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:03.958951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:03.959231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a040bcb-2b5e-4345-a39f-0abc2dce0f77]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:03.964222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0345db3-c143-4a94-a2a4-9307a014e2e2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:03.965525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:04.003836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:04.004669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:04.005098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:04.005487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:04.005874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:04.069519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ee97fbd-b0bc-4993-8d27-2f51f2ef0b0d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:04.104097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[621d48b5-e2d2-4bf0-a336-3f7398fd4a46]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:04.104097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:04.104097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:04.104097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:04.104097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:04.104097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:04.104097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:04.104097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:04.104097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:04.104097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:04.104097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:04.104097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:04.104097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:04.105041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[621d48b5-e2d2-4bf0-a336-3f7398fd4a46]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:04.105267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:04.106723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:04.108499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61dd8351-597d-44b8-a7f8-c7388ca1d526]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:04.108499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:04.108499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:04.108815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:04.108982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:04.109979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d30b9958-7bed-4b48-9a34-09e500182718]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:04.111858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bffa626b-b630-4709-bc15-c00ff2abb257]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:04.112788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16d2900e-0726-4c00-9222-ba06578aed16]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:04.141080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f943bbb0-cd63-45c6-b2a2-0c55b4e9e8a3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 654010, 'reachable_time': 36480, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202911, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0623a737-d25b-496f-91af-2b4f19a0318a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:04.152578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:02Z", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 3aa2e6eb-8c50-454a-9acd-45009aaca73e. Wait time elapsed: 11.726 Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 3aa2e6eb-8c50-454a-9acd-45009aaca73e. Wait time elapsed: 11.726 Jul 01 10:34:04.158293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:06.787329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.635s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:06.787931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:06.788660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:06.789017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:06.789301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:06.790499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbb7a077-a116-4190-add3-05ff18be81bf]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:06.808289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbf89f00-36a9-4537-a5ed-f24cd779b19f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:06.830339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[559f9991-5aa5-4ce2-b8d8-730efeff637d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:06.864055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[485abed1-7213-49fd-a840-7ffe51739e3c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202940, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:06.877691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77662a00-68bf-442f-833a-19335ecb9087]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:06.904242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ce7fd8b-b6c7-4be3-9c06-8cbbba0d3852]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:06.929921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbdee9ff-fcaa-4250-ae35-1a0a751c421c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:06.953148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38f3b6ed-1ed7-47b0-9218-dcf82eb9e4e1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:06.953708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:06.954005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:06.954810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:06.955382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:06.979329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:06.979636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eec4589e-b3e1-4811-a5d8-fa431dc37858]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:06.982516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[891395f5-28b8-48a2-9411-78ba171c8d4d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:06.983584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:07.020426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:07.021414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:07.021866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:07.022350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:07.022815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:07.068904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd1404ad-7f3a-4ddd-a317-c600cab7bfac]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.109341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[78a9baff-f0f8-44b0-90b4-e813ea204de9]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:07.109341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:07.109341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:07.109341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:07.109341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:07.109341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:07.109341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:07.109341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:07.109341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:07.109341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:07.109341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:07.109341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:07.109341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:07.110388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78a9baff-f0f8-44b0-90b4-e813ea204de9]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:07.110883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:07.112392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:07.114220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b33c7a7-eff0-4b9c-aa33-94247602f80b]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.114220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:07.114220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:07.114462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:07.114675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:07.115874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ab06e62-6bf7-42c9-a89d-cc8113cd1028]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.116851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c9d8b9d-e530-4b9d-b1ff-a6c218f4e30f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.117773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4dfe20dc-61cf-445f-88eb-6b397f06ef67]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.136835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16c26024-9865-4345-841d-eda5e340fdee]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 654312, 'reachable_time': 24192, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202959, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.139578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:07.139995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ba280d1-dc7b-434d-92a5-c1d007314a52]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:07.142490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:04Z", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:07.145492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:07.157084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 71f177e9-6b11-42c3-a122-7bbd9049bae6. Wait time elapsed: 11.795 Jul 01 10:34:07.157084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 71f177e9-6b11-42c3-a122-7bbd9049bae6. Wait time elapsed: 11.796 Jul 01 10:34:07.157084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:07.824704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.678s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:07.825273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:07.826496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:07.826889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:07.827243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:07.828508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7858a061-f324-4359-871b-aec2760a246d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.840549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97000e4d-2386-49f8-82b1-6f1c00b3c027]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.863875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac32e13c-70fb-43fa-90e3-918b3c839217]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.896229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[038c6185-8f88-4518-a14f-186c746a07d8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 202964, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.917195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d1d3760-ca64-4294-9af0-92ae6aaee6bb]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.939250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44d1e788-2866-494a-be54-9d433baf0bcf]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.960015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a7c4afc-580f-4c40-9e58-9974505d54c7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.981620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f466724-686d-4165-ac89-dba07de639ac]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:07.982208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:07.982492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:07.982760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:07.983038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:08.002317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:08.002575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbd35c58-85a7-4421-8181-0c18fc9c0387]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:08.005357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e81dc244-f797-4915-afbe-0225ce1dfe7a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:08.006408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:08.057358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:08.058035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:08.058432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:08.059004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:08.059411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:08.116027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98b87583-59e6-46c2-96d6-4196bc080f47]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:08.180783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c8093b55-fb72-4a5b-bf8f-5dbd7aab6804]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:08.180783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:08.180783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:08.180783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:08.180783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:08.180783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:08.180783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:08.180783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:08.180783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:08.180783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:08.180783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:08.180783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:08.180783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:08.182359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8093b55-fb72-4a5b-bf8f-5dbd7aab6804]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:08.184449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:08.189931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:08.193971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[316d11cc-e08d-4e86-a077-e5cf1d63bc95]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:08.195056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:08.195537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:08.195926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:08.196395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:08.197580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72729a01-1300-4b09-9bc8-67793701c49f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:08.198932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b69df56-6e36-4e47-8d29-3ff26a62ae4f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:08.200468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36acaf0d-c0fd-4f75-a903-f2acf9a969ce]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:08.231520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[478281b0-cabb-4186-9c20-e49c1789a0bb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 654416, 'reachable_time': 21012, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 202982, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:08.242395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:08.243286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[791e804c-ad9b-47f4-b458-cb0209bf7bba]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:08.244461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:06Z", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:08.247318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 3aa2e6eb-8c50-454a-9acd-45009aaca73e. Wait time elapsed: 15.821 Jul 01 10:34:08.280939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 3aa2e6eb-8c50-454a-9acd-45009aaca73e. Wait time elapsed: 15.822 Jul 01 10:34:08.280939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:11.178380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.930s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:11.179019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:11.179770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:11.180221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:11.180962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:11.183236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[303d991a-21d7-4b3c-a474-53e3ac9da65d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.212551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e3d3cfb-5d32-4efd-bb83-433b7847ca3b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.236964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[914dcde2-eeda-4ff9-8cd4-ccd01cae4e90]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.278973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42428a61-5bb3-40ee-9795-7ec3a425937f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 203033, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.311290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a47f98e-2ebf-4b25-9f0b-40d7c6feb6a3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.344305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6669da24-540b-4eef-beeb-0aa94d1db312]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.375868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b373ad88-01b9-4836-b3b7-86ac78787512]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.425137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5afbfa21-493d-4ad7-882c-bb4b843aa91e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.426046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:11.427230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:11.435969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:11.436484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:11.473675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:11.473964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[914bf42f-3583-4686-a47f-b663258aebc0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.476654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a30b62b-193e-43b5-bddc-356483b8401c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.477643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:11.530131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:11.532990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:11.534004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:11.536007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:11.536602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:11.593672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4763ba5b-1fdb-42b4-aea3-1db7fe837c3d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.638819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b1639c96-dac9-45e3-83e9-70ce6e3cf229]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:11.638819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:11.638819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:11.638819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:11.638819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:11.638819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:11.638819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:11.638819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:11.638819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:11.638819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:11.638819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:11.638819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:11.638819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:11.639684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1639c96-dac9-45e3-83e9-70ce6e3cf229]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:11.640087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:11.642345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:11.644780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a76d29f-0cad-4106-b9e4-cc5a031db183]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.644780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:11.644780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:11.644780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:11.644780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:11.646500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8962a0ba-fae1-415c-9651-bd451125d982]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.647583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db0f8447-deb2-4934-8820-d37f61987343]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.648499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0ce41e9-2570-45c3-a83a-73120d10e5cd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.672486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27c0cda4-d558-4208-8d36-69dbf840dd4c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 654754, 'reachable_time': 33968, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 203052, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.678725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:11.679050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25bd0a7a-9a5a-4a82-9d2f-a09049e0e909]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:11.680191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:09Z", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:11.683008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:11.696062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 71f177e9-6b11-42c3-a122-7bbd9049bae6. Wait time elapsed: 16.332 Jul 01 10:34:11.696062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 71f177e9-6b11-42c3-a122-7bbd9049bae6. Wait time elapsed: 16.333 Jul 01 10:34:11.696062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:12.566002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.882s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:12.566623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:12.568020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:12.568411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:12.568762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:12.570430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e5ea99a-18bc-45d1-a928-5649e9de1370]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.582373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e8ad517-34d3-4fb3-af89-430b5414a919]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.610943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[839b58f5-6453-4a12-9fb8-3b6119e41c18]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.643621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b07cd45-b0e3-4a30-ba73-553fdd07be94]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 203056, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.665604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59251ec3-7859-4678-957a-30855ad7a7d2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.688116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df11b6fb-0aa0-41ec-9cb9-9d22888c3eca]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.710887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6867b3b1-5ae9-4929-80f1-5d43d0ec902d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.734115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8406949b-b167-484c-91aa-71e39914d2a5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.734679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:12.735123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:12.736460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:12.736876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:12.755976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:12.756655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c53eddb1-e208-4ef7-a024-39c8345329ad]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.759905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66f5724e-7226-45f4-9f6f-1195f6b0cf76]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.760910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:12.826758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:12.827753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:12.828162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:12.828567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:12.829493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:12.888184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d822ef5e-63dc-480b-93a2-f6511864f884]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.934893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[69e32a0b-12a6-4b81-94c0-da051b12855f]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:12.934893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:12.934893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:12.934893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:12.934893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:12.934893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:12.934893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:12.934893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:12.934893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:12.934893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:12.934893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:12.934893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:12.934893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:12.936531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69e32a0b-12a6-4b81-94c0-da051b12855f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:12.937453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:12.939736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:12.942502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28d50239-1c75-4531-912c-d23623c1fd31]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.942502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:12.942502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:12.942502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:12.942502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:12.945033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[775bac12-5dca-42b7-a960-4b1cfb5addf5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.946516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[104500e2-702a-42d7-b1ed-1c5cd9b94400]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.947531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9485773a-0667-4385-917c-72e0b3a61adb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.968777 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b77f950a-5fe2-43a8-bc95-41df17e8a2db]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 654890, 'reachable_time': 44088, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 203074, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.978199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:12.978487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc471b19-14ac-4aef-aa8f-9492a83f6d1d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:12.980632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:11Z", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 3aa2e6eb-8c50-454a-9acd-45009aaca73e. Wait time elapsed: 20.555 Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 3aa2e6eb-8c50-454a-9acd-45009aaca73e. Wait time elapsed: 20.556 Jul 01 10:34:12.987125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:13.097365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-fc4f8eeb-7927-478d-a294-42526f7decff tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['737640e8-679f-4286-82e5-4dbeca9e7635'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:34:15.148536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:15.149124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:15.149124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:15.149297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:15.288518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:34:15.938170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.957s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:15.938462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:15.939460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:15.940528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:15.940813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:15.942031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22d4efc8-3ed9-4fe7-8bab-e2c030de6953]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:15.957078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d475031c-c9d0-43b9-a711-1c1b87f1aa38]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:15.977855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b199b876-77ad-49fc-86e2-e10e7149ec1f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.014673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[653d438a-8dca-4e39-8b78-f5583be796cd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 203124, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.036328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32283fec-46a3-4fdf-8ca9-6e98cefb28ab]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.057982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c44f517-57ef-4d11-a9e2-71bd1e482dcc]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.072751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[017136a3-b8d3-47b7-9710-b8b4d23f3868]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.096235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f64ca72-9dbb-4582-a841-96ecf4cfdfba]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.097471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:16.097960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:16.098209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:16.098615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:16.116644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:16.116926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9de14acc-ac43-4f18-89c0-cab2fe4dac27]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.120025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da318e13-9e19-4ef4-82cc-e86b91170b6d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.121033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:16.166056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:16.167400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:16.167944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:16.168469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:16.168857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:16.221097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b27d287-022a-4a52-9e88-47c044032b53]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.252660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ed03d6ca-27a6-41b3-82d1-feeb0362e9e7]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:16.252660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:16.252660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:16.252660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:16.252660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:16.252660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:16.252660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:16.252660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:16.252660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:16.252660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:16.252660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:16.252660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:16.252660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:16.253621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed03d6ca-27a6-41b3-82d1-feeb0362e9e7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:16.253834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:16.255519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:16.257281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[575a31f2-2667-492c-94ce-f3c85839d9ef]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.257522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:16.257853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:16.258115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:16.258456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:16.259559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c39305d-0ad6-4230-9a01-add0c8a78725]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.260463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1de84039-d89c-447c-8257-3eca09eb4112]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.261345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[057f8d21-a7d0-4c83-943d-c69efa66ae91]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.280378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de997a7e-6f88-481f-a587-64d120f603ae]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 655227, 'reachable_time': 26012, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 203143, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6232587d-0c96-444e-b928-99ac3cd97fb1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:16.285347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e9fe141d-f6e7-41ab-86b9-df6752ca6ecf", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.146", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "597e6f77-1ff3-4a3d-a6a5-e41898d09fed", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5516, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:be:55", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b5677ac3-0d78-46ed-b7f7-294713cf66f2", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:34:11Z", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:11Z", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:16.288607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:14Z", Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3 Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 1, update_id 2fa0acab-e4f9-4b9f-b978-025abdc865f5. Wait time elapsed: 3.191 Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 1, update_id 2fa0acab-e4f9-4b9f-b978-025abdc865f5. Wait time elapsed: 3.191 Jul 01 10:34:16.300987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:17.046152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.757s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:17.046451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:17.047390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:17.047715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:17.048559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:17.050444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6747dfae-cb54-45b0-b758-c2a2d2927890]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.060109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd26bdd5-973a-434b-a9af-b98566de6b7d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.082205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f2daef5-c9ed-4108-b8cf-b0f9f53767d5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.109666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c90a184e-18a4-4cbc-ab97-b266bf625ec6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 203146, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.130746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9c47f63-da5a-487d-9e87-dc161cc9aa3d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.150778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95085bd5-230c-4d35-a2fb-1da116f0c88a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.170927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09e19454-ba0f-495a-b45d-fcbb82017662]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.192267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[267a4082-6362-4b5e-ade3-fbd1853e1769]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.192794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:17.193071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:17.193339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:17.193619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:17.210194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:17.210930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12b64b6d-c3e7-418d-93dd-088fe5045bee]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.214126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f27ec7a-1908-43d0-be5e-2e02793a13ab]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.215069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:17.246990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:17.247941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:17.248340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:17.248731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:17.249200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:17.303917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6838faad-05b6-4c48-a6b6-992182f2b3c4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.343120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[fdfe61a3-6106-47d0-9971-2de3cfb5b5f6]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:17.343120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:17.343120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:17.343120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:17.343120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:17.343120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:17.343120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:17.343120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:17.343120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:17.343120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:17.343120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:17.343120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:17.343120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:17.344153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdfe61a3-6106-47d0-9971-2de3cfb5b5f6]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:17.344386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:17.345880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:17.348478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6a98d03-1fbf-4aec-b8f0-2d0e77d2f58a]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.349254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:17.349576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:17.349833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:17.350172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:17.351155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d849a48-d8c9-4308-9d50-cce356cf918b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.352097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2b9c0a7-e4aa-40ef-b4f9-6b1ee29fff9e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.353065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa30594e-998c-44e6-bb56-0f95af7dbd84]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.371726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03a331db-a026-4424-baa0-4704dd2edee9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 655337, 'reachable_time': 37356, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 203167, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.377902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:17.378363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6d5b492-3e89-4367-b50e-441734a6f819]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:17.379767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e9fe141d-f6e7-41ab-86b9-df6752ca6ecf", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.146", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "597e6f77-1ff3-4a3d-a6a5-e41898d09fed", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5516, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:be:55", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b5677ac3-0d78-46ed-b7f7-294713cf66f2", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:34:11Z", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:11Z", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:17.382117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:15Z", Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 71f177e9-6b11-42c3-a122-7bbd9049bae6. Wait time elapsed: 22.030 Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 71f177e9-6b11-42c3-a122-7bbd9049bae6. Wait time elapsed: 22.031 Jul 01 10:34:17.393375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:20.365596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.984s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:20.366194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:20.366765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:20.367084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:20.367384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:20.368554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47b970df-40bf-4afe-a11e-bff7c83d9dea]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.381112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15fd598b-200a-4f22-820b-dec4fcde2b2a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.403198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d247cff5-7a15-4e05-9f52-35dd43f5e166]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.433654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea6d7533-eb5e-4ca7-972d-5c78be1f0210]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 203219, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.456480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3102fe1-bb86-4279-aac4-ee1a5b397447]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.477089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6909f373-22e1-424d-bdcb-3a409ae6dc48]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.489639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52dfa9da-83b8-4069-8464-81a48fd538b5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.510716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58f8754d-6fb5-4c1e-a6f4-50134731f0d3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.511561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:20.511901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:20.512225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:20.512682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:20.530754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:20.531213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9d68aba-6f70-4fff-8d9d-a65190ddf5af]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.534107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c0004b6-bdfa-4abd-934e-978f550d26a2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.535215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:20.573910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:20.574586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:20.574989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:20.575419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:20.575976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:20.636644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88a51b4f-8cd6-4a3a-9784-9493afaac7c0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.680566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e23f97cb-1159-4e88-9a32-41e0c6c36981]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:20.680566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:20.680566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:20.680566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:20.680566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:20.680566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:20.680566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:20.680566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:20.680566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:20.680566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:20.680566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:20.680566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:20.680566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:20.681860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e23f97cb-1159-4e88-9a32-41e0c6c36981]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:20.686916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:20.689292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:20.691156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[097a7e0e-eb4c-46c2-9a8c-cb082d6b547b]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.693330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:20.693651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:20.694044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:20.694236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:20.695259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b91d5ea8-af39-45b9-90a5-1ddba3af4355]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.697066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8972b004-55d2-4143-9c52-03c15dc1b455]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.698055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f6c6ff8-1f58-4c22-ad25-3ddaa70685c1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.720902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37c9261f-887e-4164-8be5-320c61dab9bb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 655669, 'reachable_time': 30012, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 203238, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.726578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:20.726781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a19f4a21-8898-482c-bb30-d0f869997da6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:20.727978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e9fe141d-f6e7-41ab-86b9-df6752ca6ecf", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.146", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "597e6f77-1ff3-4a3d-a6a5-e41898d09fed", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5516, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:be:55", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b5677ac3-0d78-46ed-b7f7-294713cf66f2", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:34:11Z", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:11Z", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:20.736705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:18Z", Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 2fa0acab-e4f9-4b9f-b978-025abdc865f5. Wait time elapsed: 7.631 Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 2fa0acab-e4f9-4b9f-b978-025abdc865f5. Wait time elapsed: 7.631 Jul 01 10:34:20.747984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:21.564499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.835s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:21.565138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:21.565814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:21.566137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:21.566416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:21.567866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[458c05b3-9b89-40b3-8771-4b637d10aa98]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.578813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bac4f3f-5b63-4dbc-8381-c269224b5fe0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.604844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c8a093b-d0e7-49e0-a72a-053c81cfb200]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.634723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b4c7149-c76e-44d2-8f29-23155feab777]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 203241, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.655765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfe1571a-4dd8-48bc-bb18-920a605ec444]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.676489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6dd5130f-67ba-4824-9b15-011c4025ff29]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.696696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10b04f4e-7f42-46e1-8f5b-3fe50cf5d1e3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.717844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64863df9-4ed2-40fa-ac6d-7cbec52f4b38]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.718328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:21.718617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:21.718886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:21.719208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:21.734378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:21.734762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75e4ecd4-a1fc-4fc1-bada-f3371ce2fae1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.738155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25a48199-b0f3-4425-aeb5-cd80ed816a7f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.739188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:21.781459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:21.783829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:21.784237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:21.784634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:21.785027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:21.832704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[231ffe46-5b61-4074-9009-6acd2847e30b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.868072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9cf31523-226f-4ffc-8ee3-259a561f7091]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:21.868072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:21.868072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:21.868072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:21.868072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:21.868072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:21.868072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:21.868072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:21.868072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:21.868072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:21.868072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:21.868072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:21.868072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:21.869203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cf31523-226f-4ffc-8ee3-259a561f7091]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:21.869944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:21.871872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:21.874444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61fdcf7d-218e-4e8d-9ea1-4915672d565e]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.875251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:21.875942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:21.875942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:21.876270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:21.877392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ada8d8c5-f6e2-4051-9191-ad6becade780]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.878397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16fa0836-f759-4901-80e0-a8313d246886]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.879344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1c91c97-17cb-48c4-9f9f-3912b2639aaf]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.899164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0297657e-1609-4a17-9200-9fa2891002e3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 655789, 'reachable_time': 44004, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 203260, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac5c0aee-1bf1-4f4b-9f42-f9abece4810e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:21.903520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e9fe141d-f6e7-41ab-86b9-df6752ca6ecf", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.146", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "597e6f77-1ff3-4a3d-a6a5-e41898d09fed", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5516, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:be:55", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b5677ac3-0d78-46ed-b7f7-294713cf66f2", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:34:11Z", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:11Z", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:20Z", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:21.910149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 71f177e9-6b11-42c3-a122-7bbd9049bae6. Wait time elapsed: 26.553 Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 71f177e9-6b11-42c3-a122-7bbd9049bae6. Wait time elapsed: 26.555 Jul 01 10:34:21.921709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:24.726990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.821s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:24.727619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:24.728708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:24.729302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:24.729635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:24.731182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72bd89c0-48b7-4854-9e15-2d35dcba2589]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:24.745794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f704c6bf-2234-4c3b-90a5-b5ebe5f1af17]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:24.767599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[235b027c-a4aa-4228-a1ed-1a65858fc357]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:24.794802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5d15aa2-282a-49ee-9fd6-b4f829a52a72]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 203348, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:24.816819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68bd9ab3-f1c6-4113-b2e7-e429e3a8a150]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:24.837480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42f072d6-0b9e-4367-979a-e8e145613948]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:24.858510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d493406-392d-4b4c-9ebf-88e04c0bc365]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:24.880606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aade0367-aa71-46d0-a042-8ece3e469578]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:24.881206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:24.881467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:24.881742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:24.882026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:24.898021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:24.898318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c87be31-d886-4626-84f1-b69d4004f751]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:24.901283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2340b48-9b54-4db3-b949-6db97be5e0ef]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:24.902214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:24.934788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:24.939197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:24.939817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:24.940292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:24.940940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:24.978904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8da668bc-839e-4d65-b653-d6b9952f8967]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.023962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[eeedc254-edc3-4b26-b09b-ac9c30f7b97c]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:25.023962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:25.023962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:25.023962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:25.023962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:25.023962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:25.023962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:25.023962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:25.023962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:25.023962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:25.023962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:25.023962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:25.023962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:25.025066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eeedc254-edc3-4b26-b09b-ac9c30f7b97c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:25.026165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:25.028992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:25.030886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0152e59-688c-4d39-9de5-f5a4cc038ab5]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.031042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:25.032000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:25.032969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:25.033676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:25.034833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c601883d-e628-4cef-8feb-ad7df94fa823]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.035953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a689d624-7193-4b74-9b18-8e72a15efcc8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.037008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99ae370b-480d-4a33-8ed8-69f575595e0b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.062397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d935e9f2-6471-4454-b054-9f734af972c3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 656105, 'reachable_time': 41400, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 203367, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.068448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:25.068942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ff220d2-91c8-40a9-abbf-494011cb9ed5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:25.070140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e9fe141d-f6e7-41ab-86b9-df6752ca6ecf", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.146", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "597e6f77-1ff3-4a3d-a6a5-e41898d09fed", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5516, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:be:55", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b5677ac3-0d78-46ed-b7f7-294713cf66f2", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:34:11Z", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:11Z", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:25.074484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:23Z", Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3 Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 2fa0acab-e4f9-4b9f-b978-025abdc865f5. Wait time elapsed: 11.977 Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 2fa0acab-e4f9-4b9f-b978-025abdc865f5. Wait time elapsed: 11.977 Jul 01 10:34:25.087014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:25.804520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.729s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:25.805092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:25.805872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:25.806288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:25.806771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:25.808198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c148b8d-bbc7-41c3-9829-12e764ed160c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.827691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c98183db-cb90-4577-8dbd-2a55e6390fc3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.848192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c706c5f3-5f14-4b9b-b16d-f565cd1ee5b5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.875370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40c05eb1-c4a8-4075-9dba-7045c6e095b8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 203370, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.898847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8523d71-1891-4428-ae52-79ac94f0958f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.920943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a275fd3f-d7cc-4fd8-a39f-9436852b7a76]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.941468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ce8738e-bef4-4817-b2c7-f979c2070612]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.962262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[717971d7-585b-4a36-999d-8d6317a69c1f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.962739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:25.963017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:25.963307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:25.963614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:25.978674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:25.979085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a0f4604-0e10-48d6-a735-80777e50fdbe]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.982121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18973c26-d082-48ea-83c7-c77ea5fcd834]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:25.983454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:26.025756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:26.027653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:26.028052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:26.028447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:26.028827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:26.080411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ee1d0d2-e266-4392-a53f-ff299bf86c7c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:26.113874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a3f95918-8608-4a9c-b1c7-61ecef6c8fcc]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:26.113874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:26.113874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:26.113874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:26.113874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:26.113874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:26.113874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:26.113874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:26.113874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:26.113874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:26.113874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:26.113874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:26.113874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:26.114885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3f95918-8608-4a9c-b1c7-61ecef6c8fcc]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:26.115320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:26.117113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:26.119268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[603e3b0f-f249-4271-93d3-b01e23f3427e]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:26.119268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:26.119268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:26.119657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:26.119657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:26.120622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3c062b1-d56f-44c8-903d-06a88f571158]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:26.121622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c9278b5-8d61-4093-944b-d34fdb1c47eb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:26.122684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99dd1337-12ec-45fb-8894-1e01785a4d3d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:26.143143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2fea9cd-320d-4050-81ca-317ce996fc00]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 656214, 'reachable_time': 17872, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 203388, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed97df4f-0c99-494d-a5f5-80175102bd64]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:26.147513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e9fe141d-f6e7-41ab-86b9-df6752ca6ecf", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.146", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "597e6f77-1ff3-4a3d-a6a5-e41898d09fed", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5516, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:be:55", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b5677ac3-0d78-46ed-b7f7-294713cf66f2", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:34:11Z", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:11Z", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:24Z", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 21, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:26.151805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 2fa0acab-e4f9-4b9f-b978-025abdc865f5. Wait time elapsed: 13.050 Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 2fa0acab-e4f9-4b9f-b978-025abdc865f5. Wait time elapsed: 13.050 Jul 01 10:34:26.163344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:28.607058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.459s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:28.608346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:28.609394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:28.609723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:28.610922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:28.613427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25db4972-bcc0-4a90-a874-61db7a9e318e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:28.627564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[caa424ee-0d22-4c97-a464-feedf976995f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:28.651743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9563bc53-5be0-46cc-9a87-795e3cb7ff86]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:28.698728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[301ed496-a98b-420f-902b-cb5c4fda1959]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 203396, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:28.737095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6234ac7f-3025-460f-bc0a-850704cd0e7e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:28.760663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1c1b9b1-c1cb-4f39-8e3c-1f68dd8fa100]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:28.786692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40c39edb-b5a9-4214-878f-b087617688c7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:28.813736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9a80d7d-93ed-4dca-889d-655c83070d2c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:28.814314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:28.814666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:28.816401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:28.816706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:28.895823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:28.896117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76be267c-34ad-4e82-925b-78480c3b8fc4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:28.899154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8758e568-7b66-48f4-9d18-5e2222cd810d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:28.900160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:28.958235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:28.958972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:28.959395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:28.959802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:28.961286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:29.036462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42a32b5e-8dd0-48f6-86d1-1f21a43dca97]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:29.071583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[08d4ebf8-f72f-4276-955c-75c73fa290a5]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:29.071583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:29.071583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:29.071583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:29.071583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:29.071583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:29.071583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:29.071583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:29.071583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:29.071583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:29.071583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:29.071583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:29.071583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:29.072791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08d4ebf8-f72f-4276-955c-75c73fa290a5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:29.073630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:29.075722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:29.077676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[334d6bf3-f05e-492c-8c39-c3973772de5f]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:29.082127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:29.082598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:29.082746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:29.083004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:29.092899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c4325cd-589f-4d1c-a01e-4017defa315a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:29.094004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0956c37-1dca-4005-9422-1e7995e142c2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:29.095667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ba098c0-7a61-4b3c-887c-2e4f60ff2d83]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:29.118505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecab7cd8-9da9-4455-bb3a-487d566213b1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 656496, 'reachable_time': 41268, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 203458, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:29.125934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:29.126559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e146728-dded-498d-9af2-3eaa83ea7e79]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:29.130058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e9fe141d-f6e7-41ab-86b9-df6752ca6ecf", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.146", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "597e6f77-1ff3-4a3d-a6a5-e41898d09fed", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5516, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:be:55", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b5677ac3-0d78-46ed-b7f7-294713cf66f2", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:34:11Z", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:11Z", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:29.133171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:25Z", Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 22, Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 2fa0acab-e4f9-4b9f-b978-025abdc865f5. Wait time elapsed: 16.036 Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 2fa0acab-e4f9-4b9f-b978-025abdc865f5. Wait time elapsed: 16.036 Jul 01 10:34:29.155486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:29.994439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.860s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:29.995040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:29.995641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:29.996071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:29.996356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:29.997527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[260062a8-a12f-494b-9f99-f6bee0407ea2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.008112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa6b1101-79ea-44b9-b0b3-1719b087104c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.032130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eaa838bf-e79b-4887-bec3-978dc5be8f21]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.062149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ed9f007-b3c1-42c4-8c5d-3f08a2a818ab]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 203463, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.084026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a68421c-ee95-4691-abd0-63d5715678d7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.104671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86958141-529f-4671-bcd5-e8bb42f18a00]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.126692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13cf9a53-8d42-46db-bd3e-bd117071cffa]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.147660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c11826dd-ec78-431c-8fdf-d04ffe43f86a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.148342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:30.148677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:30.149003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:30.149286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:30.166067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:30.166643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6dfc8754-58c1-4ce1-a3ad-c74648b98289]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.170039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f24e929-ae88-4d62-a86f-32e0206c7f8c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.171165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:30.204081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:30.205230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:30.205626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:30.206012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:30.206392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:30.265768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8999bc71-2016-4937-a097-3925a70d4b7a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.298520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c08211f5-027d-4e34-8777-94ffc1435335]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:30.298520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:30.298520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:30.298520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:30.298520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:30.298520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:30.298520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:30.298520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:30.298520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:30.298520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:30.298520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:30.298520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:30.298520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:30.299695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c08211f5-027d-4e34-8777-94ffc1435335]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:30.300931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:30.303512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:30.307255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c1d9a8e-4959-4296-a134-cbfbccf1e1ef]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.308081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:30.308399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:30.308649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:30.309008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:30.310040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66c07d3b-9c52-4a58-9c92-247aaab159ee]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.311079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f76c06d0-9c53-43ed-84bc-bd6aa7ff1810]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.312075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[539b29b2-1a41-4bed-8084-fbae2fd05e93]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.335428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef3a9dc0-723d-4cfd-80f0-8202cbbe956e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 656632, 'reachable_time': 19252, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 203481, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.338356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:30.338774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dca9d845-bcf1-4bcb-8ff5-8dadf8a91957]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:30.340064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e9fe141d-f6e7-41ab-86b9-df6752ca6ecf", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.146", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "597e6f77-1ff3-4a3d-a6a5-e41898d09fed", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5516, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:be:55", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b5677ac3-0d78-46ed-b7f7-294713cf66f2", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:34:11Z", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:11Z", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:30.344316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:28Z", Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 2fa0acab-e4f9-4b9f-b978-025abdc865f5. Wait time elapsed: 17.247 Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 2fa0acab-e4f9-4b9f-b978-025abdc865f5. Wait time elapsed: 17.248 Jul 01 10:34:30.358321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:34:33.018794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.673s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:34:33.019406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:34:33.020045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:34:33.020370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:33.020655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:33.021804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37145e75-81b6-4cac-802b-c8c69c858836]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.034432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e00f6446-1f0f-4969-b6bd-293641f709c0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.055644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc550991-c9a9-4ce2-8c67-74774b832106]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.089831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aff701c1-bc27-4b62-8e3f-9c8566bef1d3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 203488, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.112825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e90e6823-5c22-43e1-8ab5-2add36a3d136]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.133354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d30b3c1-78a0-485e-ab34-1b3c7c0044bd]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.153672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c376b75-75cd-48db-b01c-d8ab47b9d72f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.174962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d950baf-bacc-4b05-973d-1746530840d1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.175553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:34:33.175832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:34:33.176281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:34:33.176703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:34:33.195074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:34:33.195492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f808e6f-bbbe-4755-ae7c-a72a3d2e205d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.198338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[148f1cb0-7c1e-4f80-9de2-f2db8f3cb91b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.199281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:33.239802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:33.240597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:33.241077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:33.241610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:33.242004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:34:33.303332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c59c3b71-32e2-48a6-b48d-ac1dfb986881]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.340101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8324ab58-09ee-4ec7-8a51-0a8baef03e7b]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:34:33.340101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:34:33.340101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:34:33.340101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:34:33.340101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:34:33.340101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:34:33.340101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:34:33.340101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:34:33.340101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:34:33.340101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:34:33.340101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:34:33.340101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:34:33.340101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:33.341128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8324ab58-09ee-4ec7-8a51-0a8baef03e7b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:33.341215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:33.345088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:33.347527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4fcfb3b-0b49-42c2-9a18-c12ef9e64af3]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.347527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:33.347527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:33.347813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:34:33.348007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:34:33.348955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d135256-093c-4968-a206-7a23145c9ff4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.350034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc102003-ecb1-46aa-b88c-b3b71b16e1ff]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.351241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da338077-633c-4533-9b6a-ef5901abd95e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.373427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc24fbf6-8b2c-4601-9511-fc0ff257f944]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 656935, 'reachable_time': 43764, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 203507, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2874922c-0fcf-4667-8e1d-08485e0b9f27]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:33.380497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e9fe141d-f6e7-41ab-86b9-df6752ca6ecf", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.146", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "597e6f77-1ff3-4a3d-a6a5-e41898d09fed", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.7", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5516, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:e2:be:55", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "b5677ac3-0d78-46ed-b7f7-294713cf66f2", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:34:11Z", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:11Z", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:30Z", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 26, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:34:33.389141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:34:33.406306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:33.406306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:34:33.406306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:34:33.406306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:34:33.406306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:34:33.406306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:34:33.406306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:34:33.406306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:34:33.406306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:34:33.406306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:34:33.406306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:34:33.406306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3 Jul 01 10:34:55.290165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:35:15.149922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:35:15.151996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:35:15.152482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:35:15.152891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:35:35.291252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:36:15.150488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:36:15.151218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:36:15.151421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:36:15.151726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:36:15.292524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:36:55.294908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:37:15.151572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:37:15.152233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:37:15.152233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:37:15.152411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:37:35.296246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:38:15.153352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:38:15.154012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:38:15.154136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:38:15.154357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:38:15.300932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:38:55.303163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:39:14.494071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-a626e8ed-f019-4aae-b17d-34176f8b947c tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['737640e8-679f-4286-82e5-4dbeca9e7635'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:39:14.497135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 1, update_id 12bbe2a2-c5ac-4c3a-bc81-ab8933232e5f. Wait time elapsed: 0.001 Jul 01 10:39:14.497410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 1, update_id 12bbe2a2-c5ac-4c3a-bc81-ab8933232e5f. Wait time elapsed: 0.001 Jul 01 10:39:14.497811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:15.153931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:15.154529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:15.155043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:15.156187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:15.574662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.077s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:15.575693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:15.579782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:15.580924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:15.581296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:15.582929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14b0dc25-e1fe-4236-8329-71c08be36a8b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:15.616817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a39cf79-5616-483c-8402-00f014087ed9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:15.642343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e649d032-28bd-4edf-b47c-a4686464f347]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:15.686007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd79cd8d-da66-432b-8be6-50240130925b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 207946, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:15.710472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1977fed7-a3c2-4b1c-ac5c-621a556f59ab]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:15.739155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bbc5dad-16bb-4e6c-a341-a51bb30eacd1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:15.779887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[556e1b4b-19bb-4f69-a628-11a24ee86bff]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:15.804468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[162c07c6-49a6-467a-a115-2722b9fb33b1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:15.805511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:15.805951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:15.806455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:15.806925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:15.831276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:15.832071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a82e0714-fb53-4fac-9020-04c6ea5e807a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:15.835205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a10a4e4-d73a-448c-b3ce-93ed0e9a8eeb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:15.836243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:15.891717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:15.892384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:15.892984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:15.894258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:15.895550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:15.950581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[127c0836-e0c4-4810-baaf-5eb8ae83c4a5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:16.001096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a9391942-9d88-4f0a-8ac4-5ec5eda11b2c]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:16.001096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:16.001096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:16.001096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:16.001096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:16.001096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:16.001096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:16.001096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:16.001096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:16.001096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:16.001096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:16.001096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:16.001096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:16.002111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9391942-9d88-4f0a-8ac4-5ec5eda11b2c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:16.002278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:16.005171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:16.007479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09847e63-da6a-4026-bad2-7fdcf929cd32]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:16.007479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:16.007695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:16.007930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:16.008457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:16.009556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf6fe85c-fdb9-4e28-a4b9-e52a7fd05763]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:16.011838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d653fd93-340c-4c0e-aace-ce92ded1a559]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:16.013298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c4e3161-343d-4850-ae58-37426d7e1a3d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:16.038738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[587df18e-d9aa-4ef5-afe4-57344deed0fc]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 685194, 'reachable_time': 30916, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 207964, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:16.040853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:16.041911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f43fafda-becb-47a5-a786-e8a469afcf72]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:16.043635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:35Z", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:16.046325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 12bbe2a2-c5ac-4c3a-bc81-ab8933232e5f. Wait time elapsed: 1.549 Jul 01 10:39:16.062515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 12bbe2a2-c5ac-4c3a-bc81-ab8933232e5f. Wait time elapsed: 1.549 Jul 01 10:39:16.062515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:17.873402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.827s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:17.874947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:17.876027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:17.876581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:17.877026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:17.878285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[404be920-cc2b-4551-a58f-167bc6060757]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:17.894875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[437642a0-d6a5-41f3-a8c7-449bd9fc4a1b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:17.918510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09a4c24b-1efc-4fcd-bf89-6ad1b6ae5432]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:17.953755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[498a655f-feaf-40b8-9ec2-339bc7b21418]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 207969, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:17.976512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[212b97de-323c-446c-90c4-254fafa03788]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:17.997669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8bbe6f3-a288-4b63-b0f1-520477580a69]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:18.020223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36e06bf1-217b-489e-a572-1a101b5f6823]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:18.043417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce3fde2e-dc3e-46b4-a34c-151dbe2a55fc]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:18.044035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:18.044313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:18.044592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:18.044871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(d2a79e5a-044a-40ff-a403-2bd5522145d5) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:18.060928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:18.061207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ef1ef2e-e8a4-4576-9127-6c1b4ba85493]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:18.064576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24161eb5-24b1-4a92-909e-f36785894ffe]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:18.065753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-d2a79e5a-04, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:18.101569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-d2a79e5a-04, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:18.102612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:18.104487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:18.105131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-d2a79e5a-04, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:18.105880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-d2a79e5a-04, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'd2a79e5a-044a-40ff-a403-2bd5522145d5', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:f0:45:78'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:18.165000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec49f4f0-03d2-47b0-adbe-c904504797c7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:18.213465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d328a3b0-d5c9-4e93-8637-3417d8cfc018]: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:18.213465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:18.213465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:18.213465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:18.213465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:18.213465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:18.213465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:18.213465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:18.213465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:18.213465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:18.213465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:18.213465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:18.213465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:18.214729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d328a3b0-d5c9-4e93-8637-3417d8cfc018]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:18.215024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:18.217286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:18.223999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b429c97a-1131-4c29-b683-fc0e5d589c52]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:18.225059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:18.226282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:18.226595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:18.227075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:18.229931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8e6a0ec-4f88-45ec-8a85-7ed0cc1b3fbe]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:18.232345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dee2039c-dac7-47df-9b82-e17f89220189]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:18.233701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad2435dd-79b9-480f-aa5b-9516bfda71f4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:18.267311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7cf0275-5922-4c14-9a12-69ef96a9a84b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 685421, 'reachable_time': 22940, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 207988, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:18.273484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:18.273744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb5c8e7e-7601-4b68-a132-2daf026eea82]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-d2a79e5a-04 not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:18.275100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:54Z", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d2a79e5a-044a-40ff-a403-2bd5522145d5", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "daa6cca3-c4f0-43b5-ae71-5af24a206e37", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:f0:45:78", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5504, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:53Z", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:34:35Z", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "f5dcb400-ba58-4687-b5a6-a6e94f18a0fd", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:18.278173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:18.295944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 12bbe2a2-c5ac-4c3a-bc81-ab8933232e5f. Wait time elapsed: 3.782 Jul 01 10:39:18.295944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 2, update_id 12bbe2a2-c5ac-4c3a-bc81-ab8933232e5f. Wait time elapsed: 3.783 Jul 01 10:39:18.295944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:19.086556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.807s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:19.087776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:19.088894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:19.089388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:19.089828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:19.091211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f33e6bb8-e03c-4c21-898f-4adda188e75f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.114457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a95c5e8b-55ff-4679-9293-b9d4bb31ee06]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.143633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9143db8-096e-4360-8fe9-e8d7060f375b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.208296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b02facfb-a368-4e15-af03-18d425059b0e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208004, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.246969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[166c3ea5-6f64-4e85-b452-1428c2514d81]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.290577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69488ccd-066b-4bc2-b40f-325682847d62]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.330066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be64188f-5dc5-42fa-a00b-b45682b20caf]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.345605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-bbbfe9a6-37cd-41eb-8194-5af4ccbdc510 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['737640e8-679f-4286-82e5-4dbeca9e7635'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:39:19.368793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ecc8e72-2ab4-419a-9526-3245b30f1c74]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.370098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:19.370467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:19.370823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:19.441769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b1e3b0c-cfe1-463f-81e2-e06e67a91268]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 685547, 'reachable_time': 38188, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208041, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.443025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:19.443501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:19.444074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:39:19.444498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:39:19.444974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:19.445390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:19.445841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '8a3cff40-2ad5-4ecd-8572-4922c3d45db4', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:16:1c:f6', 'admin_state_up': True, 'status': 'ACTIVE', 'device_id': '737640e8-679f-4286-82e5-4dbeca9e7635', 'device_owner': 'network:router_gateway', 'standard_attr_id': 5500, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.101', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:33:51Z', 'updated_at': '2022-07-01T10:33:56Z', 'revision_number': 4, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-8a3cff40-2a), ns(qrouter-737640e8-679f-4286-82e5-4dbeca9e7635) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:39:19.478226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:19.478630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3ec0913-00ce-41ff-a9f7-fe827e11fec0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.482293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5dac2ede-25c9-47cc-9ead-38e1d92e1309]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.483407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-8a3cff40-2a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:19.553948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-8a3cff40-2a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:19.554880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-8a3cff40-2a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:19.555478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-8a3cff40-2a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:19.556078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-8a3cff40-2a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:19.556692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-8a3cff40-2a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '8a3cff40-2ad5-4ecd-8572-4922c3d45db4', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:16:1c:f6'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:19.639203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f29401e8-1d39-4117-b89c-09f362302336]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.677657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c9d9999b-7d1c-40d7-9032-1e8ca9d60db1]: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:19.677657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:19.677657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:19.677657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:19.677657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:19.677657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:19.677657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:19.677657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:19.677657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:19.677657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:19.677657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:19.677657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:19.677657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:19.678835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9d9999b-7d1c-40d7-9032-1e8ca9d60db1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.678835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:39:19.679020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:39:19.691913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb4599f5-dd18-4e2f-b88d-a65d0654ea83]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.903632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2df43d0-0735-40a2-8aff-eac36bd1e82a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:19.915666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fda1ba85-2186-4f80-9315-766576ae57fb]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:20.106920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5743051-3fa2-4708-a724-62d2d987b828]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:20.108093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:39:20.108404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:39:20.108819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:39:20.109151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" released by "process_external" :: held 0.665s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:20.110130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:20.112324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:20.115099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0522f76-2de0-4c33-afc3-6c7bc4cfbfbf]: (4, ['qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:20.115099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:20.115099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:20.115099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:20.115099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:20.116672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7d66046-446a-45ad-9413-58f7f1ce3763]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:20.117963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b778b4e4-c1a7-4bc8-9d1d-305666f63413]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:20.119319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbfe9119-f64f-474d-89ae-da2a6ee8dffc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:20.141293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58589a85-9f68-4cf8-8c57-358322e44df3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 685547, 'reachable_time': 38188, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208064, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:20.149600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:20.150077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f762f3fd-3057-4778-abf3-1c0bdb08b3d1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 737640e8-679f-4286-82e5-4dbeca9e7635: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-8a3cff40-2a not found in namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635. Jul 01 10:39:20.152214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1882940132", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101" Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:50Z", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:18Z", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "8a3cff40-2ad5-4ecd-8572-4922c3d45db4", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:16:1c:f6", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "737640e8-679f-4286-82e5-4dbeca9e7635", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5500, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.101", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:33:51Z", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:33:56Z", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 1, update_id 4c6f1977-b1c5-4942-8626-24249542615a. Wait time elapsed: 0.808 Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 3, priority 1, update_id 4c6f1977-b1c5-4942-8626-24249542615a. Wait time elapsed: 0.808 Jul 01 10:39:20.156002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:20.878849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-455c4456-a475-47a6-924d-7f84178779d0 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got router deleted notification for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:39:21.072698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.916s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:21.073067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 737640e8-679f-4286-82e5-4dbeca9e7635 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:21.074281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:21.074706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:21.075088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:21.076444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60c302b1-f78c-44b1-862e-82e44d1674be]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.098798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[224b4133-41bf-431f-aae3-cb3ba36246b1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.121315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb983732-45af-4350-8d30-7baf9a7dbf3b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.151741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d70409aa-0027-4fa0-995c-07834e6da639]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208069, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.175077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[639b4774-55d3-4625-9812-0faf12f0f04d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.199185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d455391-5175-4294-89dc-cebd7486ad8f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.224738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[613133e7-2f7a-46da-aa83-8190293eb424]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.253306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40e52642-1548-4c1c-8d22-12f9c8e277eb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.254272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:21.254555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:21.254826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:21.283726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[358c42d2-e8ad-4969-b7bd-0caf2818386e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 685741, 'reachable_time': 21844, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208075, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.284804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:21.285232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:21.286441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:39:21.286847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:39:21.307186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2874b08c-a307-40d5-b119-04c789dc64f4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 685741, 'reachable_time': 21844, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208076, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.308892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:39:21.309360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:39:21.336855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc08fd32-5b19-4356-909a-f7de24090aa3]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.542901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47a5c67c-5615-4abc-b71b-e4a8a0620f62]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.566487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e8cbc37-e47c-46c3-84fa-29c6a695c404]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.754532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16b5e668-7eee-4abb-b1eb-29c5a4869872]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.755155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:39:21.755582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:39:21.755952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:39:21.756333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" released by "process_external" :: held 0.469s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:39:21.756875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:39:21.757239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:39:21.758078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:39:21.758794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:39:21.777321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7107ecc-9262-4540-a7a9-40889a523f83]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.814250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcee42b5-a12a-4640-b71e-fd6765b175bf]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.827175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01c939e5-405f-439a-bb6f-865d6499c24a]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.858214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02b25581-bb99-4eeb-b853-f57d67ff3241]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.858693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:39:21.858995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:39:21.859250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:39:21.859678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" released by "process_address_scope" :: held 0.102s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:39:21.859924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:21.860374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:39:21.860707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:39:21.861185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:39:21.861491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:39:21.877151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e2b79ad-8e53-4b97-883b-dbab9051884a]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.930316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85aa6c56-53c3-49ee-be7c-bfa1744b6cb0]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:21.943643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02b0b72b-8635-4448-8be5-8dcb2c100523]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:39:21 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 10:39:21 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:22.027100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f61a690e-30fc-412e-9278-25a646f95fd6]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:22.027765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:39:22.028075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:39:22.028346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:39:22.028861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-737640e8-679f-4286-82e5-4dbeca9e7635" released by "apply_metadata_nat_rules" :: held 0.168s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:39:22.029659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-737640e8-679f-4286-82e5-4dbeca9e7635 Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.haproxy Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 737640e8-679f-4286-82e5-4dbeca9e7635 Jul 01 10:39:22.030248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 10:39:22.032457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/737640e8-679f-4286-82e5-4dbeca9e7635.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 10:39:22.050328 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-737640e8-679f-4286-82e5-4dbeca9e7635[208092]: Proxy listener started. Jul 01 10:39:22.052690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 737640e8-679f-4286-82e5-4dbeca9e7635 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 10:39:22.052956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 737640e8-679f-4286-82e5-4dbeca9e7635, update_id 4c6f1977-b1c5-4942-8626-24249542615a. Time elapsed: 1.899 Jul 01 10:39:22.053226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 737640e8-679f-4286-82e5-4dbeca9e7635, action 1, priority 1, update_id 8c5bb679-a799-472f-b6cc-e9dca088cd2f. Wait time elapsed: 1.174 Jul 01 10:39:22.053467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 737640e8-679f-4286-82e5-4dbeca9e7635, action 1, priority 1, update_id 8c5bb679-a799-472f-b6cc-e9dca088cd2f. Wait time elapsed: 1.174 Jul 01 10:39:22.053758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 737640e8-679f-4286-82e5-4dbeca9e7635 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 10:39:22.054068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:22.073941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f73510e-9e15-49b5-9055-5fa4bd239b15]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:22.075628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2dfbe50d-0c13-49e9-8a57-9c68666cf3f2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:22.076552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5ad13ca-57b2-48e2-ae31-d4b0bc6d085e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:22.076975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 10:39:22.077849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2580ec5b-e7d4-446f-9c18-127b09a1071b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:22.096417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0db7c0a0-bad6-4ac2-a54e-2e46408ff3fb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 685741, 'reachable_time': 21844, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208098, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:22.097346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:22.097812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:22.118740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b4e6dbe-17c6-44f0-85eb-a1e4d2d88f6a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 685741, 'reachable_time': 21844, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208099, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:22.119698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 10:39:22.120191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:22.120514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/737640e8-679f-4286-82e5-4dbeca9e7635.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:22.120765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:22.122116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 737640e8-679f-4286-82e5-4dbeca9e7635 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 10:39:22.123144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55d24fae-944b-446f-bfe8-3dc3794e38ba]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:22.141279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[270a6a8b-3444-488c-8202-5ba659fbef8b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 685741, 'reachable_time': 21844, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208100, 'error': None, 'target': 'qrouter-737640e8-679f-4286-82e5-4dbeca9e7635', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:22.146808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-737640e8-679f-4286-82e5-4dbeca9e7635 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:22.146808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d4b4890-5e7b-4b59-a8d6-8751f922eaef]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:22.147413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 737640e8-679f-4286-82e5-4dbeca9e7635 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 10:39:22.147413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (737640e8-679f-4286-82e5-4dbeca9e7635), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:22.147656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:22.147929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:22.148228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:22.148515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:22.148763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 737640e8-679f-4286-82e5-4dbeca9e7635, update_id 8c5bb679-a799-472f-b6cc-e9dca088cd2f. Time elapsed: 0.096 Jul 01 10:39:26.342363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-e401d7c9-00bc-457b-90e6-e15d61fb4447 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:39:26.345561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 1, update_id ff0999c1-8d85-4d8b-9c1b-af5b80f86758. Wait time elapsed: 0.001 Jul 01 10:39:26.346048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 1, update_id ff0999c1-8d85-4d8b-9c1b-af5b80f86758. Wait time elapsed: 0.002 Jul 01 10:39:26.346442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:27.699042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.352s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:27.700314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:27.701283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:27.701601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:27.702055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:27.703982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6e30a42-a02c-4f08-a042-6098b219e849]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:27.715120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5abeb1b6-d137-45d2-bfd6-b1487ec750ad]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:27.740994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c14eccf-135a-4930-9ce1-e4444137ced9]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:27.775906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e37d600a-b50d-4d16-9ddb-b4fdd71f56e2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208232, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:27.797872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0a0411e-a73c-4c11-8bed-b824da625cf1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:27.821114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95a6bba2-c00c-4f36-b0cf-35ff3df43ac0]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:27.843342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e14cfc2-9ec1-45c4-b8ee-90f5470dcca4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:27.865794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d2061c9-c3a6-4605-b1bb-cae313cc3f13]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:27.866537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:27.866901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:27.867311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:27.890486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07f4c6b5-3da1-445f-8659-da688c3e869c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 686403, 'reachable_time': 32076, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208238, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:27.891541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:27.891819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:27.892247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:39:27.892508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:39:27.892833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:27.893266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:27.893556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '3c35e5b6-0e84-4ae2-a603-c2a5af89f093', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:4b:df:9b', 'admin_state_up': True, 'status': 'DOWN', 'device_id': 'dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'device_owner': 'network:router_gateway', 'standard_attr_id': 5821, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.153', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:39:25Z', 'updated_at': '2022-07-01T10:39:27Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-3c35e5b6-0e), ns(qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:39:27.913217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:27.913497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aeade95b-0879-4545-a340-39b9872f84b8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:27.916744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8659f422-3c6c-4ac5-a9c9-265c54f67751]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:27.917909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-3c35e5b6-0e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:27.918329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:39:27.919259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-3c35e5b6-0e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:27.920060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-3c35e5b6-0e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:27.920675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-3c35e5b6-0e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:27.921204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-3c35e5b6-0e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:27.921865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-3c35e5b6-0e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '3c35e5b6-0e84-4ae2-a603-c2a5af89f093', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4b:df:9b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:27.969644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8cddbfa-db56-4228-97f3-0af18d092d77]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:28.002978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e0d8988f-fc75-488a-9119-2dbd6649111f]: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:28.002978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:28.002978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:28.002978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:28.002978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:28.002978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:28.002978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:28.002978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:28.002978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:28.002978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:28.002978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:28.002978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:28.002978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:28.004188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0d8988f-fc75-488a-9119-2dbd6649111f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:28.004188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:39:28.004583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:39:28.016349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94b209b1-5a5e-4546-a18d-81861b669b1c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:28.206922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[185c1b9e-d17a-41d2-a330-e0abe64f3cbc]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:28.223022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b736b5a9-1b41-4435-a00b-61b1bbccf304]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:28.420664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc16afb6-1674-4101-99a0-6d3e15592f43]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:28.421146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:39:28.421626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:39:28.422066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:39:28.422526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" released by "process_external" :: held 0.530s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:28.423490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:28.426804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:28.431103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d632bfc6-f7e6-4eab-b607-0b4fd7cef950]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:28.432192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:28.432688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:28.433089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:28.433576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:28.434856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1533a4d7-c2c4-464f-a153-d3be4500419a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:28.435948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8152b743-2682-4a35-b5dd-d6db32f55d53]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:28.437045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[477c7660-3a84-4128-b3d9-b840ac08ec9f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:28.459760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc93a5e8-c3da-4894-97c4-2b1ee75e5b73]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 686403, 'reachable_time': 32076, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208254, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:28.470757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:28.471233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[018bd125-b95c-4ede-8806-1c289de24f29]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:28.472408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:25Z", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:27Z", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id ff0999c1-8d85-4d8b-9c1b-af5b80f86758. Wait time elapsed: 2.129 Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id ff0999c1-8d85-4d8b-9c1b-af5b80f86758. Wait time elapsed: 2.130 Jul 01 10:39:28.475823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:29.562064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-9acb1d18-3250-4dac-a8a6-01adda3ab107 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:39:32.799181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.325s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:32.799786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:32.800428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:32.800751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:32.801031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:32.805178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79d85a4c-55ce-4826-b9eb-4c58634f83b2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:32.821658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b837f7f-5584-4888-bf25-a630970bbbbf]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:32.847055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dab587b2-de02-4e0f-90c9-bf6c04afcbb0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:32.892077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73becd32-c377-4bd3-ac24-8f071d66ce03]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208311, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:32.917433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8888a8a3-3ac2-40f3-97d5-dfcfb82d2100]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:32.940557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97e57ecf-6a4b-400f-b83c-a23f52fb653d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:32.974087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[015f9f8f-c688-491b-aefa-d967549365a2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:32.997877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2d1e464-8d01-448f-bff0-562298a1bc03]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:32.998890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:32.998890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:32.999973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:33.000461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:33.020907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:33.021195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01076fff-11b8-491f-95b0-02e0abbd8b3b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.023877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c851877-3c46-49c2-9ed0-ae3669a12e20]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.024821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:33.025181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:39:33.026593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:33.027263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:33.027686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:33.028076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:33.028457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:33.077231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffdc7200-0035-41f8-ae59-b6c5cb664850]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.110761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4f9b535a-8197-48f8-a1ae-23ea7859baf1]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:33.110761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:33.110761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:33.110761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:33.110761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:33.110761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:33.110761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:33.110761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:33.110761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:33.110761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:33.110761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:33.110761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:33.110761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:33.111886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f9b535a-8197-48f8-a1ae-23ea7859baf1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:33.112168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:33.114961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:33.117900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[911bf197-ae69-41ba-8a3a-d9420fb468b4]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.117900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:33.117900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:33.117900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:33.117900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:33.120316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3657dff5-71da-414c-a6ed-ba8aa9e01b14]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.121375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af56eddc-5995-4a2f-80ad-7558edcc9036]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.122690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e95df17-dc84-4129-8982-e60a1d99cf17]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.145617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5568987-2095-4600-8d60-05b9c8dd8604]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 686915, 'reachable_time': 38788, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208329, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.152745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:33.153034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[beff1db8-cbcc-417b-ad73-23132ed9652f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:33.154049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:31Z", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:33.158794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 1, update_id a1d4b190-a2ab-4cfd-94b0-add0d5c5c6a1. Wait time elapsed: 3.593 Jul 01 10:39:33.176964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 1, update_id a1d4b190-a2ab-4cfd-94b0-add0d5c5c6a1. Wait time elapsed: 3.593 Jul 01 10:39:33.176964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:33.905247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.749s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:33.905859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:33.906659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:33.907170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:33.907492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:33.908731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd08628a-1573-45c6-8eb4-4a50e3a5d9a0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.921737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbe2327b-2e0f-4991-8c3a-cc712657070c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.943532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6fd84ab3-ad87-41af-a171-a1515e3e2ad2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.973666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10f19467-9f8d-4ba1-b123-8a3c4b1f1826]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208333, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:33.995513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68a5a525-de40-410f-8de6-0592d0f54118]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.017058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06bf0205-7104-4fc3-996e-13cf9a747e76]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.039121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e0c8a75-662a-4589-9b42-f4f4f2c925df]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.059596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af31f68e-7634-47ae-a0bf-1f4c577712fc]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.060192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:34.060476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:34.060748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:34.061031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:34.076961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:34.077281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ec321fd-bc1a-4d85-badf-a96b2361e80b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.080041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c400dfc-589f-486d-8f8f-7a2c5524f10b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.080982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:34.111320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:34.112018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:34.112422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:34.112818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:34.113531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:34.168218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30cdcdde-55a2-4b58-a3e5-71ab41300ba8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.195735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[561635fb-593a-4f02-8e98-195dae712af6]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:34.195735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:34.195735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:34.195735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:34.195735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:34.195735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:34.195735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:34.195735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:34.195735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:34.195735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:34.195735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:34.195735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:34.195735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:34.196797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[561635fb-593a-4f02-8e98-195dae712af6]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:34.197468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:34.199340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:34.202330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0becc8c2-c0bb-414e-8afb-a902f592ce82]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.202330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:34.202330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:34.202330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:34.202330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:34.203022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86884877-6dfb-4493-b87b-33e0a9db87a2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.204059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8898d9b4-50ed-479c-a22e-76ea1ee516f3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.204959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a056c09-588d-45f9-89c7-ad8717810b68]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.224549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ee042c7-61fb-4977-b29c-4b2854388c42]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 687023, 'reachable_time': 33320, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208351, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.229321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:34.229321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70c7482f-90a3-4402-b80f-0c4ab84c0649]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:34.230945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:31Z", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:34.234727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id ff0999c1-8d85-4d8b-9c1b-af5b80f86758. Wait time elapsed: 7.888 Jul 01 10:39:34.250619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id ff0999c1-8d85-4d8b-9c1b-af5b80f86758. Wait time elapsed: 7.888 Jul 01 10:39:34.250619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:35.304243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:39:36.836202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.603s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:36.836794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:36.838059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:36.838393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:36.839025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:36.840322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[509ef564-24cf-47e2-916b-2cae9ee37f56]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:36.855791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2067d176-64ef-4799-a5d5-70863e52fd00]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:36.882186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1818dd98-21a2-440b-9db0-49b68991a8f6]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:36.912900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd7b6673-7026-456d-8334-6847eca8da74]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208402, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:36.934297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f38b26a-e2ee-47f8-8e44-42b6f45df3b7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:36.954724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1b00fcd-c89a-4f64-a548-62b3d5972f27]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:36.975167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca2b8c1e-6849-4a61-a7b3-25ddbdc42d7c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:36.996108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[200c31bf-ef8f-4ffe-b6b9-5000efafe89e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:36.996610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:36.996883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:36.997144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:36.997416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:37.012240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:37.012528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3cf95c2f-0d09-4d15-8001-0c8a8c5eaa40]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:37.015189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8641dee6-c19f-4d8e-9bc6-ec9284f4295e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:37.016163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:37.051213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:37.051913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:37.052743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:37.053143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:37.053531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:37.117085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0f5715a-96cc-49c8-8182-25a9bce44161]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:37.155554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cc219e50-f3dd-4b68-ba4a-13c16180e395]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:37.155554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:37.155554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:37.155554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:37.155554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:37.155554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:37.155554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:37.155554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:37.155554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:37.155554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:37.155554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:37.155554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:37.155554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:37.156603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc219e50-f3dd-4b68-ba4a-13c16180e395]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:37.157105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:37.158748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:37.160934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87898bb0-dc5a-4207-ad4a-5ff1acbf43c0]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:37.160934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:37.160934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:37.160934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:37.160934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:37.161603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74c4b1b6-6dd4-43ec-ab97-70d030a1d3b6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:37.162585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e97f09e8-f872-4194-afb1-b141a66d6111]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:37.163579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4895a5ec-29e1-4ae0-8453-c6e2bc992a78]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:37.184801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[951b75f0-e4f8-43aa-8f76-b2e5725e7203]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 687317, 'reachable_time': 29348, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208420, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5774d38-6459-438a-9ddf-9771218e2068]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:37.189301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:34Z", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id a1d4b190-a2ab-4cfd-94b0-add0d5c5c6a1. Wait time elapsed: 7.627 Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id a1d4b190-a2ab-4cfd-94b0-add0d5c5c6a1. Wait time elapsed: 7.629 Jul 01 10:39:37.194234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:38.019772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.828s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:38.020775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:38.022223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:38.022755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:38.023238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:38.024793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[427a41cb-cc30-4776-a86e-5bf07af91948]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.043223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1942400a-e4d9-426c-beb8-ebb2a0563613]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.067995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[065a77cc-b3a9-4c89-a414-3389c736c179]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.102293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96c0023b-ecca-4d00-b889-d5fbd46a033d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208425, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.128386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2082bce-1b13-436a-a50d-9d5108d62989]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.151782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66fdf222-02d7-4d03-90a3-b21820f317ae]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.174526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ccdcaed-43f0-4573-9dcb-46f187068ec3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.197994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[083da4ff-d148-49cd-873c-f8609f472ca7]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.198655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:38.198985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:38.199517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:38.199883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:38.221404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:38.221979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b8e9e50-5730-4cfb-8939-e836da7527b7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.227572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ed3b5e0-3ced-451c-a0da-d04b5fd7464a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.229134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:38.263589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:38.264775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:38.265386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:38.266031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:38.266628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:38.340726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59c5b8ca-e2c6-4f38-9ae1-66984e20d59b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.385424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[aa827c17-d52c-41a9-8676-d26b1c662f53]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:38.385424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:38.385424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:38.385424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:38.385424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:38.385424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:38.385424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:38.385424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:38.385424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:38.385424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:38.385424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:38.385424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:38.385424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:38.386513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa827c17-d52c-41a9-8676-d26b1c662f53]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:38.387223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:38.389302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:38.391931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d0b76f76-0d78-47d4-9efa-3e1569c0d573]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.392502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:38.392877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:38.393138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:38.393532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:38.394551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df8f7faf-2921-4ab3-a425-da9da6f99521]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.395859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a9507b7-7fee-4c39-b7ee-4b2087ed3894]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.396915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61560284-f8a8-45df-8b45-535ce6753168]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.428243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3d8942c-6acc-4964-8ad3-e2072962dba3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 687436, 'reachable_time': 28964, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208446, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.441090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:38.441387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff0836b6-d861-48cd-8c55-ed752dadaeb5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:38.442424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:37Z", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:38.446233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id ff0999c1-8d85-4d8b-9c1b-af5b80f86758. Wait time elapsed: 12.099 Jul 01 10:39:38.465504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id ff0999c1-8d85-4d8b-9c1b-af5b80f86758. Wait time elapsed: 12.099 Jul 01 10:39:38.465504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:41.582656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.138s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:41.583265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:41.584083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:41.584468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:41.584811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:41.586847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9be95e8b-ed27-4021-9be8-7054dc7badc4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.603909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe150b1e-136d-467c-b041-f5c27b7a1789]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.622626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cdc7a871-a181-49a8-9cdb-d91d5e6a0a50]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.650523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3db08eae-ac81-46bf-add6-1c7b86b2d0f2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208498, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.672162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e7ddaeb-0459-4b60-871d-81214ec73bd9]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.693945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ebe5507c-b27d-43b7-a7c6-df947f868250]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.715468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c166b259-a0ec-4c12-9000-dd108359cb2e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.735727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee74ae56-9a15-439e-9664-31f7fb654d9f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.736289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:41.736568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:41.736837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:41.737112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:41.753333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:41.753631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd21cf1d-e17e-4f4c-b361-133cf8446450]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.756365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ba16520-af8e-498a-b817-9e1ee9252b0a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.757319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:41.812551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:41.813305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:41.819568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:41.820126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:41.820802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:41.864724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40138a7c-a2f0-409c-ae51-b814256874bb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.901199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[44cbbb74-658a-4bfc-a80e-fb8508dda036]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:41.901199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:41.901199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:41.901199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:41.901199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:41.901199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:41.901199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:41.901199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:41.901199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:41.901199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:41.901199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:41.901199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:41.901199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:41.902185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44cbbb74-658a-4bfc-a80e-fb8508dda036]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:41.902508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:41.904291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:41.906690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f9a277d-7a6d-4c6f-ba4b-b2914d2f7a0a]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.906690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:41.906690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:41.906690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:41.906690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:41.908236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30dc06f1-d587-4a77-b168-87067f1223a6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.909171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3f65943-45c7-4579-99ac-81f859fbd662]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.910103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[560a730f-1d0c-4d0b-ab25-eeed3a41f3a9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.929348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2dfb2325-1d21-4a2d-b239-182951029eb4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 687791, 'reachable_time': 32552, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208516, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2848a6cc-a500-4ff2-a2a4-2d7fcc1dd909]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:41.934609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:39Z", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id a1d4b190-a2ab-4cfd-94b0-add0d5c5c6a1. Wait time elapsed: 12.372 Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id a1d4b190-a2ab-4cfd-94b0-add0d5c5c6a1. Wait time elapsed: 12.372 Jul 01 10:39:41.939739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:42.642317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.707s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:42.642903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:42.644889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:42.645288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:42.645642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:42.646845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22c7e505-c6a4-43d8-ad9a-4e9f841402b0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.660176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2141fa49-e837-4238-b467-98614d20b642]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.683060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64794ff8-5f81-4809-a341-0403bb44adcd]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.710976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[017547dd-9c4e-4967-9da9-cd1530e4249d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208519, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.735569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fea4e32-9273-44fc-b179-c2456a1f24bc]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.760615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1108c657-06c4-4ed4-b4db-e328a0b2fa9f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.781961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd5ba0d6-32bd-442f-8f17-cf97c8812a85]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.803545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93dc7c94-481f-4afa-97be-32a6ed77a97e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.804285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:42.804632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:42.804969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:42.805321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:42.822173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:42.822759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[120dd274-0157-4cff-a27c-0250c50e5f3f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.826104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c476a9d6-d32e-4831-ac72-4f9224cef06c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.827082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:42.854722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:42.855907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:42.856409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:42.856906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:42.857400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:42.916837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9243dfdb-782b-472b-a6d7-d62c57eb475f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0d4fc8f8-8575-45c0-b579-1ae38f59a451]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:42.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:42.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:42.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:42.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:42.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:42.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:42.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:42.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:42.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:42.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:42.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:42.950943 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:42.952142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d4fc8f8-8575-45c0-b579-1ae38f59a451]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:42.952321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:42.954154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:42.956835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[beb594a3-3d9f-466c-9227-0c27a58e57cb]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.957812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:42.958223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:42.958458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:42.958883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:42.961445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fefa3d4e-82d0-43fd-801f-b294d1d97a4f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.963487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59993acc-b9df-4627-9e27-8ee03232ad91]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.964591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3459e8b5-fa86-48b0-a90d-e860dd9e6d6f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.985446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[303c3405-7610-4446-9323-a6e07fae3169]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 687897, 'reachable_time': 33024, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208537, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fce95f24-485a-4a70-8914-2f2ae67cd53b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:42.992557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:41Z", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:42.996329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id ff0999c1-8d85-4d8b-9c1b-af5b80f86758. Wait time elapsed: 16.651 Jul 01 10:39:43.008446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id ff0999c1-8d85-4d8b-9c1b-af5b80f86758. Wait time elapsed: 16.651 Jul 01 10:39:43.008446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:45.489687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.494s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:45.490310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:45.490934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:45.491258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:45.491613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:45.493064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b34f88a3-ba05-42e0-b91b-529d3e7b2393]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.506211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e38dc57-2b23-4a72-bd4c-e9e67bb5c883]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.530352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0fc3e6b-37df-4175-a184-2c539a172796]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.560389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45d14852-9281-420c-a93d-881958e4ccc9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208543, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.583103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87920782-b3bd-49b8-923c-a9acf32603c7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.613059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e6f4474-8084-4212-96d1-c56dc7f26722]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.636338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bfad8bc-cffb-48ee-97e8-348751fe8b5a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.672626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[857eec2e-e311-4f41-b638-e9dc4a1b9721]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.673649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:45.674169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:45.674890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:45.675180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:45.695604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:45.695900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5988fc60-7024-443a-b5ec-2a8b59dc8d3b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.699501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e5d0592-fecf-4291-8375-fd690e316c8d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.700469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:45.735112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:45.736301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:45.736959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:45.737559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:45.738156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:45.824251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4dcde6a4-22a4-44cc-b5e9-8ca0d0c31b9e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[127db05a-013b-4429-a217-e9f7760cecb0]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:45.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:45.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:45.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:45.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:45.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:45.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:45.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:45.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:45.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:45.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:45.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:45.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:45.867315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[127db05a-013b-4429-a217-e9f7760cecb0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:45.869028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:45.872583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:45.876368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad3fc1eb-2149-4346-805b-9978ccc0cf44]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.878028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:45.878498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:45.878913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:45.879497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:45.880607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2472c136-6a4d-4a30-a8c0-31c23fdb6080]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.881725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70a1f6c5-d4a1-414d-ba73-90197fc2704a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.882765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09d500f7-881e-4dd1-bdaa-06fe91122dd9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.907249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9ccb701-e5c8-478a-990a-5107cb1e2c22]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 688182, 'reachable_time': 19120, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208563, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c885dec8-2d63-4bc4-a3c2-c9a2e4bb6bfe]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:45.912098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:42Z", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id a1d4b190-a2ab-4cfd-94b0-add0d5c5c6a1. Wait time elapsed: 16.349 Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id a1d4b190-a2ab-4cfd-94b0-add0d5c5c6a1. Wait time elapsed: 16.349 Jul 01 10:39:45.917697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:46.791652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.879s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:46.792331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:46.792638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:46.792956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:46.793237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:46.794679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1dd8cc07-635e-4b61-bad1-2140b56894bd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:46.819644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea938aa7-1201-4261-83fe-0a3f3b137125]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:46.879273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f3103b8-77cc-4977-a1a9-567f3dbdb953]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:46.943569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[caa9a98d-1cbb-4aec-a882-22d008a06860]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208587, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:46.981813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62998a20-7039-437e-b68f-cba471c51d74]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.058170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5167664b-694e-4546-bc04-d01a999866a9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.105048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30892bf6-c40d-42da-a885-2411ecc7789e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.131672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b73d5b8c-f1e1-414b-8156-c5c60aa5afe1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.132616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:47.133073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:47.133514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:47.135247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:47.154458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:47.155124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d827ac0-8a86-45ac-a4a2-4550a82e4109]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.158026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a40284a-5089-4085-b38e-a0c3bbc15f45]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.159217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:47.200595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:47.202378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:47.202969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:47.203583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:47.204421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:47.276612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b887808-9e34-46fa-b3da-e78760a9a919]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f48b519f-a4c3-4d80-9628-6b5021ca2a7a]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:47.327593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f48b519f-a4c3-4d80-9628-6b5021ca2a7a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:47.328800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:47.333288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:47.333288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:47.333288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:47.333288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:47.333288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:47.333288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:47.333288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:47.333288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:47.333288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[924e709d-831f-4bc3-9898-bee42388dae3]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.333288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:47.333288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:47.333288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:47.333288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:47.334849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e8b5f09-1d94-48df-9b5a-0b7ffaa2a575]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.336190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[693fad51-9459-43b3-ace7-7e1ba90fd60d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.337787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[587bbf68-50c8-45c2-8686-0162fe5ee94e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.368691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce843390-83a8-4bcf-9ba7-017a1e059f65]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 688320, 'reachable_time': 20300, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208630, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.379543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:47.379919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6050ce02-f509-49b2-8afe-b925f6c83e78]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:47.380957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:46Z", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:47.385608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id ff0999c1-8d85-4d8b-9c1b-af5b80f86758. Wait time elapsed: 21.038 Jul 01 10:39:47.406954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id ff0999c1-8d85-4d8b-9c1b-af5b80f86758. Wait time elapsed: 21.038 Jul 01 10:39:47.406954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:48.026699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ce28fdf0-58a1-47ce-bf07-4cc58c08c769 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:39:49.706722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.324s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:49.707341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:49.708423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:49.708746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:49.709023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:49.710201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7338a637-7cc1-4de6-9f3d-9d08e56bf8c9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:49.731219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02ad201d-1d28-4a31-bd92-9ee5dfa4d45a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:49.752563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06681972-cfe9-4d5d-b372-7d6150891fec]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:49.782306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a0207a6-030a-4a52-bbec-596caf7f506f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208639, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:49.804371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[502d20d6-940b-42fb-a04f-7928c0d64cb1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:49.825801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17dbee0a-270b-4b73-a91f-3b8ad2e56a8b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:49.845661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbc8f22c-ec51-45b1-aec3-5e8f804cfabb]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:49.867144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d9283ea-21bd-41f9-8da5-08a990d8ad21]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:49.867887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:49.868129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:49.868435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:49.869104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:49.883805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:49.885059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57d02d67-d003-486c-8351-8862a006bdde]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:49.888048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7d060e5-103f-4838-aadf-5538ca32d8c3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:49.889346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:49.924092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:49.924924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:49.925544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:49.926094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:49.926639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:49.992269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8564573d-d901-472d-9424-6a77d2044e40]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.028345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4c9ecca8-d161-429d-b6a2-44ac9b3b3bd6]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:50.028345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:50.028345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:50.028345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:50.028345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:50.028345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:50.028345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:50.028345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:50.028345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:50.028345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:50.028345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:50.028345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:50.028345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:50.029314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c9ecca8-d161-429d-b6a2-44ac9b3b3bd6]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:50.029603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:50.032540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:50.034721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3eba4d68-0901-48f0-b111-19446eb48e97]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.034721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:50.034934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:50.035167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:50.035686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:50.037017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40ef8dd4-8840-46b2-b87e-b592bc3a36cd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.038055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be96741f-9e0d-4faa-b79f-14c933da25c6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.039094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b86fa52-b2a3-4dc4-a99c-ce5de8ab9afb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.059929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[362599c1-3e5f-4478-8385-fd520644840a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 688604, 'reachable_time': 44144, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208657, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba12aeb4-631e-4fc4-bb5c-f8a78f0149e3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:50.064256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b9cee1c-097f-4a9f-9c23-e1f109476a02", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.123", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "52408fd1-5b79-4ba8-830c-aa9f4b2387f7", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5840, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3a:68:37", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "79e4a37d-b60d-4d50-97fa-3d1268b9bef9", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:46Z", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:46Z", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:50.067563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:48Z", Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3 Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 1, update_id 584f284e-2b79-449f-8444-e1807f535bee. Wait time elapsed: 2.038 Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 1, update_id 584f284e-2b79-449f-8444-e1807f535bee. Wait time elapsed: 2.038 Jul 01 10:39:50.080363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:50.811866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.746s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:50.812514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:50.813137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:50.813462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:50.813764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:50.815333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1c1b540-90c2-4052-b448-407af456d09e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.836845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ccf5e78-41c5-4ca5-8d03-4741702724b7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.860065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75d13952-bc2f-4880-bace-f31ae564ce5d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.889837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8caa01b-d364-4460-86b4-9e643069b274]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208660, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.910427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d75ed086-5024-4411-bc3b-72cf19dc2ed5]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.931216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25525aef-4e64-4a51-b6e2-21991deaca14]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.952732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5783705-a526-4caf-a16a-c048b0d1d641]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.974621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[836b5905-1e3c-4961-9811-9da612583ef3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.975175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:50.975475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:50.976164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:50.976455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:50.991792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:50.992071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[077757d8-15b5-4dac-b3cc-9e889b8909b3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.994827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed246a4f-a375-4393-9a7d-75ca9fae1aaa]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:50.995937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:51.031504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:51.032188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:51.032585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:51.033077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:51.033350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:51.090955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21b24abb-ac31-4328-9e27-ff4820504963]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:51.126724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c0014c58-6753-4ec2-8479-79df1f51e048]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:51.126724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:51.126724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:51.126724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:51.126724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:51.126724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:51.126724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:51.126724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:51.126724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:51.126724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:51.126724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:51.126724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:51.126724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:51.127793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0014c58-6753-4ec2-8479-79df1f51e048]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:51.128251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:51.130179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:51.134880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[270496dd-1e66-4f4a-96f2-da3996584925]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:51.134880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:51.134880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:51.134880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:51.134880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:51.134880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6373c8b-403c-462d-b0b7-fe91709488ac]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:51.134880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ad40f44-a9c1-4fad-8642-7a415709d1ae]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:51.135678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eebf5daa-5b8e-41e6-a636-528f7a40b835]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:51.155228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b38d4cdc-da17-465a-9db1-3fc31a89212b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 688715, 'reachable_time': 30760, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208679, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:51.157311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:51.157665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29560409-4fc6-497b-9ea6-381c84850fb1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:51.158732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b9cee1c-097f-4a9f-9c23-e1f109476a02", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.123", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "52408fd1-5b79-4ba8-830c-aa9f4b2387f7", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5840, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3a:68:37", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "79e4a37d-b60d-4d50-97fa-3d1268b9bef9", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:46Z", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:46Z", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:51.161482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:49Z", Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id a1d4b190-a2ab-4cfd-94b0-add0d5c5c6a1. Wait time elapsed: 21.598 Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id a1d4b190-a2ab-4cfd-94b0-add0d5c5c6a1. Wait time elapsed: 21.598 Jul 01 10:39:51.173625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:53.845593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.685s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:53.846233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:53.847484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:53.847946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:53.848238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:53.849387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c3e0961-3bb5-4f62-8615-5c5aac91010e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:53.867011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[750075b8-135a-4cf1-ab29-1f003f0fc93c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:53.889635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4cec188-a8fa-4e24-9ad8-e73d3a8b7aa9]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:53.923574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91e62456-da44-45a4-846f-5def382368b1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208729, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:53.945980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a53ac4b-a983-41ac-a92a-424178912c80]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:53.967522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6cb587e8-115c-4d87-a725-434d61ee9233]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:53.987900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d689a500-7777-4bdb-b5ec-9fcfb05364b9]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:54.008330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a640c37-8ce0-406b-8e6f-cf12a36252fd]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:54.009053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:54.009338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:54.009613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:54.010105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:54.026150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:54.026822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99d84f3f-1383-4a28-a09d-f14f82220330]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:54.029529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9a2a066-9bb9-4e55-85c3-58e9863be905]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:54.031033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:54.070820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:54.072380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:54.072841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:54.073292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:54.074115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:54.127996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00cd93f2-581f-4084-8153-167f2fcfe526]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:54.162362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f8fce08e-d3df-4015-a5b9-12a0940226ee]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:54.162362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:54.162362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:54.162362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:54.162362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:54.162362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:54.162362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:54.162362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:54.162362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:54.162362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:54.162362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:54.162362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:54.162362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:54.163397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8fce08e-d3df-4015-a5b9-12a0940226ee]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:54.163605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:54.165546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:54.168241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3cf811d-b944-4ffc-ac21-0dbe7a5e4a49]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:54.169092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:54.169424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:54.169689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:54.170036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:54.171094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5da0dbc8-eacc-4ab3-8aec-0bcd57c608fe]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:54.172258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c29098ac-2d32-4609-acd5-83a43ef4b612]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:54.173198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ec1d9f1-8b3b-4e97-b19a-bae288b06bf3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:54.192257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4584ea4-cbbb-4e69-b814-96e46dc76f18]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 689018, 'reachable_time': 23564, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208749, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbc4fc4e-dde0-43c5-b804-3d5b331787ee]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:54.198849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b9cee1c-097f-4a9f-9c23-e1f109476a02", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.123", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "52408fd1-5b79-4ba8-830c-aa9f4b2387f7", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5840, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3a:68:37", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "79e4a37d-b60d-4d50-97fa-3d1268b9bef9", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:46Z", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:46Z", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:51Z", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:54.204079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 584f284e-2b79-449f-8444-e1807f535bee. Wait time elapsed: 6.170 Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 584f284e-2b79-449f-8444-e1807f535bee. Wait time elapsed: 6.171 Jul 01 10:39:54.216492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:55.017632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.819s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:55.018258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:55.018898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:55.019223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:55.019526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:55.020933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9db39704-4eb4-4eff-b0af-1d601a280c80]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.032054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a478d70-def4-4caa-b44e-5916630bf15c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.060107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[923e4304-6754-4baa-b39b-a136ad572e1e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.088869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf4157b0-5cb4-4ec4-b334-dc8473075590]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208752, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.110911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48f88d53-2576-4ad5-8903-6d087ad63719]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.135788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[064d441a-353d-4737-821d-4c2ad40dec9b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.156205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc6c00ec-f524-46d4-8083-4fdcfd876620]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.176405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2cb2ecef-cdb0-4fa6-8426-46ef65480512]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.176951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:55.177270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:55.177536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:55.177810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:55.193254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:55.193584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d1020f0-e80e-4cc9-a486-47dd0b2648ac]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.196948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e77f0be8-9101-4561-80f3-23c127f20709]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.198090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:55.235302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:55.236331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:55.236779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:55.237172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:55.237581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:55.299444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[495504bf-3eb8-493e-bdbb-058df7140ebb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.330336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4fef891e-9a2c-435c-a013-4c82b93376b4]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:55.330336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:55.330336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:55.330336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:55.330336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:55.330336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:55.330336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:55.330336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:55.330336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:55.330336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:55.330336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:55.330336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:55.330336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:55.331342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fef891e-9a2c-435c-a013-4c82b93376b4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:55.331571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:55.333426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:55.335898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d3d2682-28b9-4000-81cd-c26dafb08d94]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.336042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:55.336278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:55.336539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:55.337414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:55.338515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5561ce55-3eb9-4891-aec2-c4cbeb81cad5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.339504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b9d9fcd-ac56-4b05-8f4a-b4604bc33be3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.340617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[150f7bd9-1c81-4fbe-b36c-e00e66301648]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.359750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e87969b-0712-4753-8380-f0194e69958f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 689135, 'reachable_time': 34652, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208771, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.363023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:55.363838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1b66f46-0d49-4f54-9f92-d494d43bd9a6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:55.368082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b9cee1c-097f-4a9f-9c23-e1f109476a02", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.123", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "52408fd1-5b79-4ba8-830c-aa9f4b2387f7", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5840, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3a:68:37", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "79e4a37d-b60d-4d50-97fa-3d1268b9bef9", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:46Z", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:46Z", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:55.371232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:54Z", Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id a1d4b190-a2ab-4cfd-94b0-add0d5c5c6a1. Wait time elapsed: 25.808 Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id a1d4b190-a2ab-4cfd-94b0-add0d5c5c6a1. Wait time elapsed: 25.808 Jul 01 10:39:55.384404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:58.306218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.935s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:58.307031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:58.308149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:58.308643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:58.309017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:58.310264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[143dd8b1-191d-4538-ae37-47f7e874a874]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.328985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[810e2ec4-4b70-4b90-9b48-a437f48d4837]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.354847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e95deab0-3b41-453a-a4a4-53feb4fb80b0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.387221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b5e4952-6adf-4583-bef0-c632773f7e22]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208826, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.409434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db6157ea-7fb8-40b1-940f-ca617085246c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.452226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed0d5b9a-0a33-490a-b9b3-0ee44b1f52cd]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.483640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08bfeafd-c10f-47c2-90d1-53e34c14f2df]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.507288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51b8b03f-0384-4456-a9c9-cb6e42519005]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.507988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:58.508350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:58.508469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:58.509399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:58.527315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:58.527626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6366cc3f-60a0-40eb-876f-4ada83125e1c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.530585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55acb536-0850-4e16-9b53-e70c3602fe8a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.531555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:58.566346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:58.568477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:58.568981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:58.569441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:58.569922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:58.640686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4577eacb-b4f4-412b-917a-be649ace5d02]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.680430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[23d79012-aec4-40a9-953c-f103893b2096]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:58.680430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:58.680430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:58.680430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:58.680430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:58.680430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:58.680430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:58.680430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:58.680430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:58.680430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:58.680430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:58.680430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:58.680430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:58.682312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23d79012-aec4-40a9-953c-f103893b2096]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:58.683523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:58.685164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:58.687175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49022f40-d348-4e42-bba2-ac92b9854be8]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.687175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:58.687175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:58.687175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:58.687566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:58.689568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60360b0b-4ba8-4b9d-aef7-a0bc6c1082f6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.690621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cafcf0c6-2018-41ff-9e96-8158e33f7186]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.691724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1055d673-436c-4b67-8759-e8bd18c869ce]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.714479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87676105-51ef-4560-955c-275d42994439]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 689464, 'reachable_time': 30524, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208848, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.721313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:58.721631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9242f42c-4910-4eba-b6d6-c663c457685b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:58.722710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b9cee1c-097f-4a9f-9c23-e1f109476a02", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.123", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "52408fd1-5b79-4ba8-830c-aa9f4b2387f7", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5840, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3a:68:37", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "79e4a37d-b60d-4d50-97fa-3d1268b9bef9", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:46Z", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:46Z", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:58.725914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:55Z", Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 24, Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3 Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 584f284e-2b79-449f-8444-e1807f535bee. Wait time elapsed: 10.699 Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 584f284e-2b79-449f-8444-e1807f535bee. Wait time elapsed: 10.701 Jul 01 10:39:58.737718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:39:59.511998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.781s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:39:59.512620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:39:59.513503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:39:59.513929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:59.514277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:59.515518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e69c8670-3c06-41c9-9704-599962ebd328]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.529201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d9600f7-3baa-47a0-a1d1-d85556c759bc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.552212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97a4b32d-a568-47d1-8950-9aa9ce41beb0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.582391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35f2222b-8ec5-469c-a7ed-aebacca9072c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208852, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.604214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01aebf51-298e-48c9-9607-7d7f8b9cbd6f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.624265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ae9c6cb-1043-44d4-bef1-c5b4f09a711e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.644328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8751aac7-7abc-4504-9deb-94338a66521d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.666139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47cb7469-2a84-411b-9a8f-82372c1b1d8f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.666687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:39:59.666968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:39:59.667239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:39:59.667588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:39:59.684335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:39:59.684627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2ac6e57-4afb-4379-8b15-f69feae0a2b1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.687212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f01112ab-daf5-411f-8f7e-6b8d1475b546]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.688274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:59.728432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:59.731090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:59.732770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:59.733708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:59.734309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:39:59.776825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01b1eae1-692c-418b-b9df-38bb0d8ae2f3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.818994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2cac985d-e007-46ef-96fd-4e08efbcaef8]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:39:59.818994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:39:59.818994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:39:59.818994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:39:59.818994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:39:59.818994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:39:59.818994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:39:59.818994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:39:59.818994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:39:59.818994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:39:59.818994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:39:59.818994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:39:59.818994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:59.820171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2cac985d-e007-46ef-96fd-4e08efbcaef8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:59.820829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:59.823385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:59.825417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b56b18c-e945-4c6f-b391-20ff1fca23b0]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.825417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:59.825417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:59.825417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:39:59.826977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:39:59.828626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d13a3d3-2314-4454-acce-aaee2a170e0e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.830074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbe290fa-950d-4d3f-9e41-a4c3051e8a71]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.831232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81540fcc-ec90-48b3-9b3e-00b17356863b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.852121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23ded409-e31d-4c06-bb93-a940d475a9ff]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 689584, 'reachable_time': 17684, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208871, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.857147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:39:59.857245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bd87f16-3e23-436c-bd67-fb1f0f5c16ce]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:39:59.858201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b9cee1c-097f-4a9f-9c23-e1f109476a02", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.123", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "52408fd1-5b79-4ba8-830c-aa9f4b2387f7", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5840, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3a:68:37", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "79e4a37d-b60d-4d50-97fa-3d1268b9bef9", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:46Z", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:46Z", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:39:59.861116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:58Z", Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 27, Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 584f284e-2b79-449f-8444-e1807f535bee. Wait time elapsed: 11.832 Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 584f284e-2b79-449f-8444-e1807f535bee. Wait time elapsed: 11.832 Jul 01 10:39:59.875863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:40:02.721653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.861s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:40:02.722313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:40:02.723104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:40:02.723468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:40:02.723788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:40:02.725424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73b53c47-d802-4987-8b78-8e1f3fe5e921]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:02.742191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe72f5ac-a96e-49eb-a800-c2f951237ac5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:02.765814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1851ee3c-f514-4e61-bfeb-df0bb444d879]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:02.796881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72fd8ea1-b406-42c1-9258-c158dffea2b0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208922, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:02.818355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ce4ac93-f253-4aa7-9096-0decd95f6613]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:02.840372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e48b3396-9f8d-4d15-8747-fadd1fcf03d3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:02.861474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd91b361-31dd-4fa6-9d08-5a51678aba44]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:02.883232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19aab716-ed72-4001-b732-1f1693c53431]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:02.884493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:40:02.884826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:40:02.885153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:40:02.885519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:40:02.901539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:40:02.901915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32f94b33-eb70-4e61-b5e0-733cd9adf244]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:02.904686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76e8347e-8cd8-4324-b15c-95b67323b0a2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:02.905617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:02.949546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:02.950217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:02.950613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:02.951026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:02.951449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:02.995896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99e5ab64-9246-4631-9300-4a4632115732]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:03.044512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[beb7ecf8-8944-4c05-a8e8-435b8a026d3f]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:40:03.044512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:40:03.044512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:40:03.044512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:40:03.044512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:40:03.044512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:40:03.044512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:40:03.044512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:40:03.044512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:40:03.044512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:40:03.044512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:40:03.044512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:40:03.044512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:03.045507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[beb7ecf8-8944-4c05-a8e8-435b8a026d3f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:03.045821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:03.047827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:40:03.050377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[189d6db3-3975-4607-a9b6-e043fa4a8bdd]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:03.050377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:40:03.050377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:40:03.050377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:40:03.050377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:40:03.050843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e00a04b-fe2f-4c95-b05b-66230c9c9ac7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:03.051938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[235361ef-4c77-48d0-932a-1a7069a4d8d2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:03.052952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[304e060c-5b8c-4a9d-b3ad-ea042931ea45]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:03.073020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b57c4c7d-4911-4633-8d3c-c8f7f4e83d4f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 689906, 'reachable_time': 15776, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208940, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:03.075081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:40:03.075655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[438b610d-c834-478d-b851-3f3bb1f31972]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:03.076878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b9cee1c-097f-4a9f-9c23-e1f109476a02", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.123", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "52408fd1-5b79-4ba8-830c-aa9f4b2387f7", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5840, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3a:68:37", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "79e4a37d-b60d-4d50-97fa-3d1268b9bef9", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:46Z", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:46Z", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:40:03.080716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:40:00Z", Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 28, Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 584f284e-2b79-449f-8444-e1807f535bee. Wait time elapsed: 15.058 Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 584f284e-2b79-449f-8444-e1807f535bee. Wait time elapsed: 15.058 Jul 01 10:40:03.092467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:40:03.900988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.814s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:40:03.901582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:40:03.902202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:40:03.902527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:40:03.902811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:40:03.903990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79b72908-75f7-48c0-a351-2eb279aeb3ee]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:03.918670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[674a692c-edcd-4e20-af56-963792227d72]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:03.939793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99e20579-c1b4-49b1-b6ab-045b920143a3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:03.967626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5485ece3-a6eb-4af1-963b-6c4b8ab6b499]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 208943, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:03.989818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[180e1921-e229-4990-8c75-a3ea8858d219]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.011468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[232f75ef-b6ee-4be3-9299-b05c063c36df]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.032735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[980dc387-5213-4f7b-9a3e-e07c64ccc5a9]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.055256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e49488a-eed4-45d8-b5a7-43d13234f833]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.057074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:40:04.057414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:40:04.057752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:40:04.058063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:40:04.075123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:40:04.075454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de469216-2793-47ff-8b5f-5d6ccd4bca59]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.078095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[485c68f8-55ff-431d-acfa-9290fcbbb77a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.079092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:04.119223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:04.120006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:04.120647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:04.121552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:04.121674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:04.185650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42ac5a19-9122-4440-82d5-9cb849b8e3cd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.215108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8a39f6b5-53e5-4ce3-aaca-0dc4b4060d7b]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:40:04.215108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:40:04.215108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:40:04.215108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:40:04.215108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:40:04.215108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:40:04.215108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:40:04.215108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:40:04.215108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:40:04.215108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:40:04.215108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:40:04.215108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:40:04.215108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:04.216245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a39f6b5-53e5-4ce3-aaca-0dc4b4060d7b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:04.216350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:04.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:40:04.220773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4de76ec0-4e36-41eb-8d66-5380c1370c69]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.220773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:40:04.221232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:40:04.221501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:40:04.221967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:40:04.223260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f2bdfab-c40d-478d-a73b-9d96a541ab00]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.224213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7b55426-0847-41cf-8b92-6815c5e4474f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.225163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[541e30c0-53ec-46c9-a9db-bb1cf4de5fbb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.250589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02fae627-3e76-4602-b285-3e80a964e6aa]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 690023, 'reachable_time': 25460, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 208961, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.257416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:40:04.257892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1da512e9-6fb0-4287-b17e-c88b5c83d248]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:04.259605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b9cee1c-097f-4a9f-9c23-e1f109476a02", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.123", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "52408fd1-5b79-4ba8-830c-aa9f4b2387f7", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5840, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3a:68:37", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "79e4a37d-b60d-4d50-97fa-3d1268b9bef9", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:46Z", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:46Z", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:40:04.262672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:40:02Z", Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 584f284e-2b79-449f-8444-e1807f535bee. Wait time elapsed: 16.233 Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 584f284e-2b79-449f-8444-e1807f535bee. Wait time elapsed: 16.233 Jul 01 10:40:04.275291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:40:07.110796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.849s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:40:07.111597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:40:07.112271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:40:07.112596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:40:07.112975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:40:07.114345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[199f6c5d-9212-4fdc-aa36-6e519b1ef7c7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.129682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[766dadb2-5a6d-4f56-9349-5d981e935944]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.151652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8298d3b7-11d2-42ba-8d69-f8adc793b273]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.193256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03bc3bd9-b073-4235-b78a-5e7cccc90a06]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 209011, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.215137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce1029ba-9c14-45ae-859e-30a1b9d9258a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.237016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92f8f55e-d7ba-4e21-8b8b-fb775a42beb8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.259986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d5ccc92-a20e-43b9-b750-ef4143ca6415]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.281429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[124cd8dc-a168-4bd5-8990-5b7233b8d8f2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.282100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:40:07.282439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:40:07.282776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:40:07.283118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:40:07.300344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:40:07.301282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd904ec1-ed38-44f8-9bf0-372a340c7989]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.304158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[069a1294-b431-4e7f-8d7a-5d1c44bed7e2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.305830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:07.344263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:07.345015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:07.345478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:07.345926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:07.346370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:40:07.398587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d5106be-4f54-4e25-b2b2-5b622f6d4459]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.454579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2889f5b8-ba73-4fbe-a732-3f0853463bf0]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:40:07.454579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:40:07.454579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:40:07.454579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:40:07.454579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:40:07.454579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:40:07.454579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:40:07.454579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:40:07.454579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:40:07.454579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:40:07.454579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:40:07.454579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:40:07.454579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:07.455880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2889f5b8-ba73-4fbe-a732-3f0853463bf0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:07.456317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:07.458224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:40:07.460870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f8fd3b6-60a5-4a6b-8afa-ed47cf9b436a]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.460870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:40:07.460870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:40:07.460870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:40:07.460870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:40:07.463144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[430823fb-6209-4754-84b3-39b509144970]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.464227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[739d98a5-ea85-4073-b636-36a5bcd6073d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.466117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfd65453-dd1a-44f0-a184-3c5872726a8a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.488320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59cbe857-8adc-4407-b82d-578421f996d4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 690345, 'reachable_time': 23960, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 209031, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.496784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:40:07.497045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ea2e3bb-efdc-4bdc-8916-a67b2eeddd15]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:40:07.498060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b9cee1c-097f-4a9f-9c23-e1f109476a02", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.123", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "52408fd1-5b79-4ba8-830c-aa9f4b2387f7", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.5", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5840, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3a:68:37", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "79e4a37d-b60d-4d50-97fa-3d1268b9bef9", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:46Z", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:46Z", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:40:07.501035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:40:05Z", Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 30, Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:40:07.515623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3 Jul 01 10:40:15.154822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:40:15.155466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:40:15.155575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:40:15.155786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:40:15.306548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:40:55.308311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:41:15.155663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:41:15.156389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:41:15.156389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:41:15.156866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:41:35.309223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:42:15.156094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:42:15.156759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:42:15.156759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:42:15.157069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:42:15.310860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:42:55.314062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:43:15.156698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:43:15.157451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:43:15.157451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:43:15.157451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:43:35.315178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:44:15.157321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:44:15.158023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:44:15.158172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:44:15.158494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:44:15.318697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:44:49.560583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-3b14ae64-ac3c-4818-80f7-9d57e8bee3fd tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:44:49.561510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 1, update_id 29ea559f-372a-4e83-8604-b9456aa2839f. Wait time elapsed: 0.001 Jul 01 10:44:49.561996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 1, update_id 29ea559f-372a-4e83-8604-b9456aa2839f. Wait time elapsed: 0.001 Jul 01 10:44:49.562764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:44:50.518251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.955s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:44:50.518526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:44:50.519551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:44:50.519885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:44:50.520171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:44:50.521445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d57a0a1d-3744-46f6-a859-7c869606a7f5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.534933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5d8c25a-0f28-4404-97d4-d3c38c79794d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.556338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e1fc120-1f97-4e14-b4ae-759cb54feb91]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.583498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4120623-9364-4bbe-9999-3e10fa79458d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 213495, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.604655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cdbf9937-ca90-46f7-914f-ff490fb1143a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.629926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ca67841-f100-4b6b-b6e0-1eb44ca8d35c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.650491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd8ab9b0-b33d-484a-8976-0b894b004d74]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.672505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02289d38-889f-42c5-883f-be8511e932be]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.673068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:44:50.673343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:44:50.673667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:44:50.674031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:44:50.688848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:44:50.689134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1cef930a-07ac-4b23-b855-a74fec5286db]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.692011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b011c784-1bf0-47b4-9dc5-f39ba5c53fc1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.692974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:50.731417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:50.731760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:50.732224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:50.733154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:50.733647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:50.816594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0a91425-7090-4d03-9936-3904d067fc21]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.863662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a799f6c5-51cc-4b04-897a-4267ecf67747]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:44:50.863662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:44:50.863662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:44:50.863662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:44:50.863662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:44:50.863662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:44:50.863662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:44:50.863662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:44:50.863662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:44:50.863662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:44:50.863662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:44:50.863662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:44:50.863662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:50.865056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a799f6c5-51cc-4b04-897a-4267ecf67747]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:50.866483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:50.869975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:44:50.872855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a2c0a55-cd51-4ce5-834b-4251065aa648]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.872855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:44:50.872855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:44:50.873507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:44:50.873507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:44:50.874618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1860ea6d-026a-4f37-8a5d-d1aa6a764b53]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.875559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33a74d44-2ffa-4ac3-8db6-bc8de977a935]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.876576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcdcddcd-9bea-4781-82bd-a58e5e0116bc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.910451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e428ae2b-fd6d-4dac-aab1-8d44bc2829ba]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 718684, 'reachable_time': 42744, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 213513, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.916426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:44:50.916870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d78e118-c604-4bf0-a703-88186843a0c6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:50.918185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:40:10Z", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 33, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:44:50.921546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 29ea559f-372a-4e83-8604-b9456aa2839f. Wait time elapsed: 1.358 Jul 01 10:44:50.937489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 29ea559f-372a-4e83-8604-b9456aa2839f. Wait time elapsed: 1.359 Jul 01 10:44:50.937489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:44:53.030028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.110s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:44:53.030632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:44:53.031185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:44:53.031536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:44:53.031818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:44:53.033574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2a3d030-72d8-4f53-bf23-064a1d8f17db]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.049945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b71609f3-cad0-4a97-b032-afe7fae825d0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.070379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fff4fac-fcff-4021-9bc9-d325244f1a2e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.099418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4239592-5aeb-4ebe-bf2e-446e069a82be]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 213519, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.120347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20c76822-01f1-4c27-ab73-1d17eecf6c9f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.141246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec1a6b12-8ddd-4e4a-9d4f-2f0720f8c975]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.161920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e93558b-ccc0-4153-b548-8c20ea150556]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.182444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85aa5a05-17d9-4da6-bb1d-1c4c36054493]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.182992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:44:53.183270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:44:53.183608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:44:53.183884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:44:53.199874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:44:53.200183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3621e39-8137-4831-ad7f-661c09fd48da]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.202911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f069d929-8101-4b9d-89d5-002122305bbf]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.203969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:53.248703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:53.249555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:53.250023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:53.250689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:53.251148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:53.309644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5afce03d-1b1c-4f85-899e-5fbdf751f587]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9c638ae2-303d-45ef-bbc8-85f4286ed9cf]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:44:53.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:44:53.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:44:53.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:44:53.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:44:53.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:44:53.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:44:53.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:44:53.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:44:53.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:44:53.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:44:53.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:44:53.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:53.339762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c638ae2-303d-45ef-bbc8-85f4286ed9cf]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:53.339972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:53.341752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:44:53.346693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8ba7e66-2217-49a0-8e89-b20f90c80796]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.346693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:44:53.346693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:44:53.346693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:44:53.346693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:44:53.347738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7a3146b-e4f5-41d6-ad63-9ff7e4742ace]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.348537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa2aca6f-27bb-4a37-b096-aa1a89de4f4c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.351729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21b8ead1-b282-4394-ab5b-0fd593026590]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.373838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02bcbc6a-8872-42c9-b24d-97b605d448b2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 718936, 'reachable_time': 41028, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 213538, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.379867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:44:53.380126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[401da3fc-9cba-414b-bafd-b2029bc95fc2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:53.381186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:44:51Z", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 34, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:44:53.384766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 29ea559f-372a-4e83-8604-b9456aa2839f. Wait time elapsed: 3.823 Jul 01 10:44:53.399346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 29ea559f-372a-4e83-8604-b9456aa2839f. Wait time elapsed: 3.823 Jul 01 10:44:53.399346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:44:54.145764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.761s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:44:54.146430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:44:54.147067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:44:54.147496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:44:54.147851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:44:54.149458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c70f1ff4-a45b-4c33-a2d1-1fb39da1d2f9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.165645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39023cef-19ef-4b42-aae5-26d2087bffda]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.187720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5657bf17-c9d4-45c0-9551-cde8fe6e10a6]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.218752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3cb07c4-e35e-40dc-a387-5ad717b85811]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 213542, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.242161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a510302-97c9-454b-8b2b-2c9c677d6ac4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.264387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25615f64-5284-4f76-a87c-67c1d117799e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.287480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4418256-c7ea-423e-9a8f-59db4800d143]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.311921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0a5187c-1f2d-40ab-8794-e36c2576ee74]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.311921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:44:54.311921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:44:54.311921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:44:54.311921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b3a177d7-7128-4866-8979-2c7529619f12) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:44:54.331528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:44:54.331822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6b4c4a1-5dbc-42e6-a96d-481a9a719041]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.334813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a0965b1-7519-41ec-8abb-57821b378ffb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.335805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b3a177d7-71, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:54.381635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b3a177d7-71, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:54.382616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:54.383204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:54.384991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b3a177d7-71, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:54.385577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b3a177d7-71, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b3a177d7-7128-4866-8979-2c7529619f12', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:cf:dd:25'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:54.451544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f1c1091-9a2c-4198-a261-7b9e4d4038e4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.492252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[97d1c12c-c3b0-4048-98d8-1d87e670d7b5]: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:44:54.492252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:44:54.492252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:44:54.492252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:44:54.492252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:44:54.492252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:44:54.492252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:44:54.492252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:44:54.492252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:44:54.492252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:44:54.492252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:44:54.492252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:44:54.492252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:54.493870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97d1c12c-c3b0-4048-98d8-1d87e670d7b5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:54.494529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:54.497528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:44:54.503288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c49ce5c-2235-46b5-be7a-5324dc322588]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.504373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:44:54.504754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:44:54.505061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:44:54.505483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:44:54.506775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29860f80-2e36-479b-856b-f1121728120d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.508306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e471493-ecf6-437d-86dc-683b70131a5b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.509365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb4ff470-11f8-4822-818f-f5d4b8dafc1e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.535673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1846c8b0-481e-4fbd-bf4b-c80709076136]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 719048, 'reachable_time': 42256, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 213561, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.538266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:44:54.538819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83926d57-c662-4229-b773-77f0a5086b47]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b3a177d7-71 not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:54.540142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:28Z", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b3a177d7-7128-4866-8979-2c7529619f12", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "14240855-d8fa-49fb-9b3e-e1e45f76b133", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cf:dd:25", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5825, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:27Z", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:44:53Z", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 35, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "259fc4e7-ef81-4f52-aaf3-19a147187f7f", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:44:54.546386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:44:54.565854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 29ea559f-372a-4e83-8604-b9456aa2839f. Wait time elapsed: 5.000 Jul 01 10:44:54.565854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 2, update_id 29ea559f-372a-4e83-8604-b9456aa2839f. Wait time elapsed: 5.002 Jul 01 10:44:54.568683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:44:55.320494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:44:55.529346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-265b01f3-326e-4ba5-b920-480fda9b9b9b tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:44:56.646730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-113fa130-d688-4276-9129-d9bdabc3b63b tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got router deleted notification for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:44:56.732903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.166s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:44:56.733281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:44:56.734907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:44:56.735240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:44:56.735656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:44:56.736713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4083fa0b-416c-43f1-a57c-dcb4d0641f25]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:56.757083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1ff045f-a78a-43ef-a204-de27a1338b1c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:56.780825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80d339c0-b09b-4196-a64c-fe0f49810a2d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:56.812589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3663491-1bce-46fd-b5bc-47356ddadb3c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 213614, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:56.834523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c6e888b-b2fc-4e01-bff1-ebb616cd4dd9]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:56.856723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d7f68f6-1b86-431a-8456-62a772bab1f2]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:56.878187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6057c47-b683-4fb6-ac13-e2fca49a7450]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:56.901079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9212a0ab-7a29-40b5-b6e6-0f2e34bc5fe6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:56.901808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:44:56.901949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:44:56.902185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:44:56.926927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca890121-b49a-4d8f-906a-29dee515b1ef]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 719307, 'reachable_time': 15364, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 213620, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:56.928813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:44:56.929100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:44:56.929542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:44:56.929807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:44:56.930318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:44:56.930588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:44:56.930886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '3c35e5b6-0e84-4ae2-a603-c2a5af89f093', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:4b:df:9b', 'admin_state_up': True, 'status': 'ACTIVE', 'device_id': 'dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'device_owner': 'network:router_gateway', 'standard_attr_id': 5821, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.153', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:39:25Z', 'updated_at': '2022-07-01T10:39:30Z', 'revision_number': 4, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-3c35e5b6-0e), ns(qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:44:56.958253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:44:56.959226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff2d2c9b-891a-41e0-a5c4-2522efd047c9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:56.963048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a16efe06-32fa-4e17-a1b8-d8a455cf23c5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:56.964144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-3c35e5b6-0e, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:57.010352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-3c35e5b6-0e, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:57.011312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-3c35e5b6-0e, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:57.012035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-3c35e5b6-0e, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:57.012888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-3c35e5b6-0e, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:57.013505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-3c35e5b6-0e, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '3c35e5b6-0e84-4ae2-a603-c2a5af89f093', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:4b:df:9b'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:44:57.084014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79235f64-29b7-40df-8fed-d2a38f42b336]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:57.133124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5942ae2d-985c-4fd6-81f9-20f62e9da072]: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:44:57.133124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:44:57.133124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:44:57.133124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:44:57.133124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:44:57.133124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:44:57.133124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:44:57.133124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:44:57.133124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:44:57.133124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:44:57.133124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:44:57.133124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:44:57.133124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:57.134171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5942ae2d-985c-4fd6-81f9-20f62e9da072]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:57.134171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:44:57.134469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:44:57.156942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b7bcbd7-2af6-4552-ac6b-b997ed9ed4cb]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:57.342461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7efa855-084c-478c-ba23-f66e21339b14]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:57.372056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[186d1d9f-71d6-4248-a250-dfb77e55ef66]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:57.567311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bb6a648-d80b-40d4-8a2c-1177d37309e6]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:57.568275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:44:57.569110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:44:57.569476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:44:57.569947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29" released by "process_external" :: held 0.640s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:57.571013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:57.573648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:44:57.582064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88641923-3c77-4879-8770-08d694979989]: (4, ['qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:57.582064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:44:57.582064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:44:57.582064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:44:57.582064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:44:57.584556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[524878d8-6fc1-47d1-bc5a-dfd36cce174e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:57.585057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab005b43-1633-4205-bb93-0ad9185871f2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:57.587058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3371977-5526-40a0-a1f7-148fd1cf44c5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:57.616680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46715d94-635e-48df-97ec-40af53d2aca8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 719307, 'reachable_time': 15364, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 213639, 'error': None, 'target': 'qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:57.626967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:44:57.627222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af344c9a-f7c2-48c7-be52-6370eb9e19af]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-3c35e5b6-0e not found in namespace qrouter-dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29. Jul 01 10:44:57.628658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-595590564", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153" Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:24Z", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:44:54Z", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "3c35e5b6-0e84-4ae2-a603-c2a5af89f093", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:4b:df:9b", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 5821, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.153", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:39:25Z", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:39:30Z", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 1, update_id 6abc6744-9adc-43a5-aa8f-d484d4d244ee. Wait time elapsed: 2.099 Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, action 3, priority 1, update_id 6abc6744-9adc-43a5-aa8f-d484d4d244ee. Wait time elapsed: 2.099 Jul 01 10:44:57.633710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:44:57.664564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.034s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:44:57.665291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29 was not found. Performing router cleanup Jul 01 10:44:57.666833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[191a5335-4a8e-4fe6-bd18-71ec9d2fdffd]: (4, ['qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:44:57.667932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:44:57.668280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:44:57.668609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for dce7d2f5-8409-4c7b-8cb9-9f7bfe1f4a29, update_id 6abc6744-9adc-43a5-aa8f-d484d4d244ee. Time elapsed: 0.039 Jul 01 10:45:00.509336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8a32b44b-679d-4f12-a4e6-8a908466aaf1 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['e4c7d557-c62d-4089-9e91-1907e88396ea'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:45:00.510996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 1, update_id f392b58f-0a12-48c1-8b37-17f75d881270. Wait time elapsed: 0.001 Jul 01 10:45:00.511248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 1, update_id f392b58f-0a12-48c1-8b37-17f75d881270. Wait time elapsed: 0.001 Jul 01 10:45:00.511767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:03.356296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.844s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:03.356832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:03.359282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:03.360125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:03.360415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:03.362846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8435c4a-497e-40b6-8093-53d18fdaf87b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.379982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71e2de11-a4b3-4a56-86fc-9e5caa481379]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.405182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ace24006-2c8e-4c34-b686-60493716174f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.455292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[227fd700-d6bd-4186-ac5f-246b56e73382]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 213711, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.476657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86e63618-c4d5-4dd4-8782-5467282c999d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.498824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fea2af6b-1e23-42a0-a2a5-60d638054497]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.522560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c61aa658-ebb8-4fe3-af5e-ddd80617699c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.526531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-89ffe796-da6d-4a60-9811-8e1c25dae5ff tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['e4c7d557-c62d-4089-9e91-1907e88396ea'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:45:03.547886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6dab1890-af0e-46ca-9b90-043b45cfe830]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.549007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:03.549280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:03.549543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:03.549819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:03.568681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:03.568940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa023669-27e2-4f3a-b4ee-669ec37384f1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.571820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6460cd96-2cb3-4689-80a9-40101ec994a5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.572891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:03.573362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:45:03.574200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:03.574974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:03.575495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:03.576107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:03.576568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:03.630223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[652313a1-8efa-4550-88b8-33c05d571f17]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.667587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1988e501-bfeb-4605-8610-d35bb6cb6b83]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:03.667587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:03.667587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:03.667587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:03.667587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:03.667587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:03.667587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:03.667587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:03.667587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:03.667587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:03.667587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:03.667587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:03.667587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:03.668978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1988e501-bfeb-4605-8610-d35bb6cb6b83]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:03.670769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:03.673032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:03.676553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bceb97e-0e36-4d39-999e-84022ed17dae]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.676553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:03.676553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:03.676553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:03.676553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:03.680884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf2bc4de-559e-45bf-851a-f359dd6418fe]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.682146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83cc5ee2-4fcb-453d-ba4d-f79fa244c844]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.684161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ede3e025-d136-48f2-a360-92a42e709ebd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.706106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30d66500-121f-4241-ae77-47dacbac08d6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 719971, 'reachable_time': 44040, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 213730, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.714056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:03.714322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e60425e-aa9a-4344-9249-3df530825219]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:03.715444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 1, update_id 16e1a644-d2e0-4ad3-b35b-f349d80109ef. Wait time elapsed: 0.190 Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 1, update_id 16e1a644-d2e0-4ad3-b35b-f349d80109ef. Wait time elapsed: 0.190 Jul 01 10:45:03.721372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:05.285995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.569s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:05.286566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:05.287034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:05.287390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:05.287679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:05.289498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6c0bcc6-e272-4b48-94f0-a94b20b69eb7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.314671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b353610-f4e7-4446-bb7b-c4a1f2ad293d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.363718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[307a02e2-b98b-43c2-95d9-8277a2181126]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.420920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a94dd1d-863a-423b-9846-30305f488f6f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 213754, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.459334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2afbc1a-0cdf-4539-8cba-f7f82f549fb5]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.494383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20a6f025-ebe6-44be-bd19-926f1de0a8b7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.552281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76a803b9-3b21-4bfb-bbd3-cf8f846cc714]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.628984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c27b9a2-a0c1-4358-9d55-4a4e2917fb43]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.632121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:05.632529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:05.632970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:05.633615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:05.679278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:05.679566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2855730a-9606-4fa3-8fad-58b6eb6a5adc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.682422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1886e5fe-bf47-4195-a9ce-a666b8f5b694]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.683395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:05.730528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:05.732539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:05.733423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:05.733979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:05.735076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:05.836614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9df2797-b7ba-4f30-855b-a2a664989a70]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.909367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[38693398-16cd-4f06-8b22-7ab1d38ec6fc]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:05.909367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:05.909367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:05.909367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:05.909367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:05.909367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:05.909367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:05.909367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:05.909367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:05.909367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:05.909367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:05.909367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:05.909367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:05.912125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38693398-16cd-4f06-8b22-7ab1d38ec6fc]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:05.913214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:05.918597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:05.921468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22a4fc91-5125-4625-a5f9-92dfd4da621e]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.921468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:05.921468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:05.921468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:05.924232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:05.936223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bff75492-7474-4deb-85dc-f3d399112ed1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.937457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01e1e6b0-97af-4b34-9bcc-08ee01338b81]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.938442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4afb42e-2649-4a6d-a5ce-16471e27da6a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.963207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99447625-8263-4ee4-9d87-a6cc223adc94]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 720168, 'reachable_time': 17472, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 213803, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.971206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:05.975467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3395a560-ca3f-4e9e-9423-f5b197bda9db]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:05.976885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:04Z", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:05.981761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:05.996564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id f392b58f-0a12-48c1-8b37-17f75d881270. Wait time elapsed: 5.472 Jul 01 10:45:05.996564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id f392b58f-0a12-48c1-8b37-17f75d881270. Wait time elapsed: 5.472 Jul 01 10:45:05.996564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:08.140237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.157s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:08.140765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:08.141718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:08.142501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:08.142885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:08.144282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e8af655-6c8a-430a-9e07-e7dbd46b3e39]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.170926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7ee80a6-208a-4088-b7b7-f824c5d69ef1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.195686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d40d55f-4a8e-48fa-8ef1-20e04e625430]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.231709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3689961d-3432-4318-9331-62c056d872c8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 213811, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.257266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e328b8b-8dca-4100-a7ad-16c2da4ea2ca]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.284537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93a93be2-930c-44f2-b3b5-3e232cd60ce9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.309490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b11eba1-bc57-452a-a0fe-b57d715d07f5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.332972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bef6a9f0-71b6-4505-b519-34dbfaed52a6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.333678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:08.334029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:08.334390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:08.334752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:08.351044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:08.351462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05fe20a8-c65d-429a-b359-2b2e8f8e754d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.354836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1be59d6-dd8f-422e-a175-92630fac7fe5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.356132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:08.385894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:08.386673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:08.387136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:08.387643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:08.388430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:08.452302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00d61bc9-0c37-4c28-94d0-703071865e4d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.504749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[74f24de5-6bd8-4666-8a5e-ab0dfafb9142]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:08.504749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:08.504749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:08.504749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:08.504749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:08.504749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:08.504749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:08.504749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:08.504749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:08.504749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:08.504749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:08.504749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:08.504749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74f24de5-6bd8-4666-8a5e-ab0dfafb9142]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:08.505991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:08.507889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:08.510602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[365f1a7a-3145-45f0-9cf9-a2478ef8cfc7]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.510602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:08.510602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:08.510602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:08.510602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:08.511319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8ca9527-2a5b-4ece-9cdf-4bf4b4d61b0d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.513152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e789333-eebd-433c-8d55-52f4ffe680ab]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.514455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa0fa0c8-dcb0-4398-b5fc-0a22906d82da]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.537747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0cd4a5d-a20f-47b9-9225-3c47d9eac605]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 720449, 'reachable_time': 31520, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 213829, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.539989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:08.540489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01236d7d-7f67-4bad-b7b4-e293d704d2de]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:08.541756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:06Z", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:08.547050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:08.559292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id 16e1a644-d2e0-4ad3-b35b-f349d80109ef. Wait time elapsed: 5.021 Jul 01 10:45:08.559292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id 16e1a644-d2e0-4ad3-b35b-f349d80109ef. Wait time elapsed: 5.021 Jul 01 10:45:08.559292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:09.277146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.729s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:09.277787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:09.278569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:09.278975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:09.279323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:09.280634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07b40ac8-9c6d-4421-ab66-829c89d67fb2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.299308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdac4afa-b17b-464d-88b0-dd131b7d4934]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.321189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90511172-1427-4162-a569-fde6584d31d5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.356519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7db9de2-d890-477c-b476-8ff132d6f7d7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 213832, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.391245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81ca56cc-7aec-4b88-938c-6154174ff0b5]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.416672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d53aa31-ad4a-4cdf-9433-05650ef10681]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.455325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9ade186-aa21-453a-8a5c-f46c189433dc]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.488065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ab9b8d6-989c-4482-a7f4-500c67d30919]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.488669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:09.488994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:09.489271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:09.489555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:09.512447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:09.512730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4481f49e-c8a6-40cb-9a30-d7f7845f5c60]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.515935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c5de8b6-8fea-4300-8f19-53022d0492b1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.517233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:09.559236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:09.560106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:09.560513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:09.560910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:09.561301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:09.619773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6a7bd90-afc8-4ba2-a6a4-4b93b74c842b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.659574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0895d3ee-502c-43f3-ac59-f63ed81f61e8]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:09.659574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:09.659574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:09.659574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:09.659574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:09.659574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:09.659574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:09.659574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:09.659574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:09.659574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:09.659574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:09.659574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:09.659574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:09.660823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0895d3ee-502c-43f3-ac59-f63ed81f61e8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:09.661450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:09.663765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:09.667646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83660524-3e97-48fd-a60e-c8c5025bf192]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.668575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:09.668954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:09.669212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:09.669624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:09.670698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b57167b8-eef0-4836-8aa3-8fef34f743e9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.671696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd1b4785-d622-4bc7-bad1-3fbf518d0f8f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.672611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12745297-0790-4edd-8d87-75b6a748962a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.692776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[442a678e-b442-4820-a187-bda2f06d94ab]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 720562, 'reachable_time': 44516, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 213851, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.699659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:09.699950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc06f484-8542-4885-8cf8-5c1c9dcb30cc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:09.701143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:08Z", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:09.705948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id f392b58f-0a12-48c1-8b37-17f75d881270. Wait time elapsed: 9.192 Jul 01 10:45:09.719316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id f392b58f-0a12-48c1-8b37-17f75d881270. Wait time elapsed: 9.192 Jul 01 10:45:09.719316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:12.386431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.684s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:12.386974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:12.387794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:12.388174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:12.388461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:12.389681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7bf3be8-c6dc-4785-b198-8bcf70b39d2a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.404568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21c79b27-4cb3-4c06-9238-98058aecbbc4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.426589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9297a357-ed38-416a-9233-179247d8dfc9]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.458037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21f8f5fc-0ca1-4cf9-a1ee-3146a8e1ca87]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 213901, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.479006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95fce4ee-e4a4-47dd-ad30-c03c5540c355]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c54f2972-2899-4b53-9833-b215b6cddaed]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.522427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[556efe76-488b-4593-8b43-f370cb4a9849]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.543337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40d46ba2-0586-4bf2-a383-d373e1272217]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.543873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:12.544168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:12.544428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:12.544703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:12.561148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:12.561440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c7896e0-b943-4a6d-8d31-67cd8fa1ce51]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.565214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5eaee3ca-ebe9-4846-921a-687a76393a2d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.566577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:12.608871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:12.610050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:12.611875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:12.612433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:12.612896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:12.660050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[526cb111-4717-48a5-bdf3-44944b32600c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.692779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e07ffc79-7e93-412b-99bc-c6da76ac1d76]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:12.692779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:12.692779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:12.692779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:12.692779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:12.692779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:12.692779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:12.692779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:12.692779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:12.692779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:12.692779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:12.692779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:12.692779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:12.694440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e07ffc79-7e93-412b-99bc-c6da76ac1d76]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:12.695403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:12.697367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:12.699661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c646201b-beb0-405b-8cc1-54e2b6906c04]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.699661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:12.699661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:12.699661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:12.699661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:12.700900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[176b40df-b800-4f0b-b8b8-811c4e557f9f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.703415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39f8c3dc-cc33-4893-b7c6-20c4344d1bd5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.703415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3efd4323-7538-4746-9529-96cf6553ddf6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.727432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7bfdfc4-ff44-4cd3-bcaf-1238135d04ed]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 720872, 'reachable_time': 15980, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 213921, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.733988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:12.734474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f02dbeaf-da10-4491-aebc-b4fd0147cad1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:12.735975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:09Z", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:12.740208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id 16e1a644-d2e0-4ad3-b35b-f349d80109ef. Wait time elapsed: 9.210 Jul 01 10:45:12.757551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id 16e1a644-d2e0-4ad3-b35b-f349d80109ef. Wait time elapsed: 9.211 Jul 01 10:45:12.757551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:13.554913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.817s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:13.555707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:13.556283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:13.556604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:13.556880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:13.558054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6940c82-856e-4c8d-830d-9e518a8ad852]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.572131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfeb9323-e70f-4b88-9edc-993afe52a8e2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.594286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a80ced5d-16cc-4197-be4f-a301aef3b7ee]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.626060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57b08ecb-857d-4a8c-9454-cd73180e07b0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 213925, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.647926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6bb139fc-1a80-4da4-ba63-bff9f90dbded]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.671212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[559ed312-117b-4b3e-a1c0-b999aac910a8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.693160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[797ae2ef-177c-4ff9-9709-32bd397afd5d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.713117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4150139-1634-42ef-ba4b-f48f3c254a52]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.713709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:13.713992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:13.714261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:13.714541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:13.731065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:13.731346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c0f0832-3f4a-4302-b17b-55efbc956175]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.734003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35fc595f-a1f6-480c-9326-32d56004e086]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.735000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:13.774721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:13.775742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:13.776261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:13.776648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:13.777030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:13.827417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fea81f33-a9aa-4865-9d8e-3571c12366ca]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.870329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ec5603a0-ccc2-4caa-aa1c-d0dae74c753e]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:13.870329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:13.870329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:13.870329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:13.870329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:13.870329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:13.870329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:13.870329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:13.870329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:13.870329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:13.870329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:13.870329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:13.870329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec5603a0-ccc2-4caa-aa1c-d0dae74c753e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:13.871683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:13.873751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:13.877447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1d23aa0-c270-42a8-9a36-9e69eb09581b]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.877447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:13.877447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:13.877447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:13.877447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:13.877447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60c00453-9924-4864-961a-4abe8a589135]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.878672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c26ac1e3-ef70-4413-a9f8-d9be7574f091]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.879285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[872a6827-222e-4c63-b709-1b307b4f9779]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.899147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f81d581b-e8ac-4f64-b275-a1e616f3e39f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 720988, 'reachable_time': 37568, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 213943, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.904517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:13.905185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[249460f0-9073-4f99-b818-ffae47184e96]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:13.906326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:12Z", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:13.910116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id f392b58f-0a12-48c1-8b37-17f75d881270. Wait time elapsed: 13.399 Jul 01 10:45:13.926182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id f392b58f-0a12-48c1-8b37-17f75d881270. Wait time elapsed: 13.400 Jul 01 10:45:13.926182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:15.158473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:15.159133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:15.159133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:15.159349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:16.950687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.040s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:16.951326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:16.951929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:16.952506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:16.952785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:16.953970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b166dd4e-7571-42b7-a96e-65ec0757cd90]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:16.971424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6880ba0-8e7a-44d3-911b-11994e1b8008]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:16.992285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89cd6907-11ec-4c9e-ae1f-a9311a374631]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.021538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f04b1fe5-7918-4029-8a96-0ce7a1d293bf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 213993, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.042449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ff17673-0bbd-4932-ab5b-bc1cad9085e2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.063973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ab167cb-ad82-4d23-9f96-0b857b7ae65e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.085262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db9207ba-1e05-4cc1-afef-e99c8dafaf78]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.106922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67ee512d-589e-4690-9524-2f0262dabe5b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.107638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:17.107835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:17.108103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:17.108760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:17.147323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:17.147910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e45fad5d-23c7-44ab-9f56-c5af73ed931d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.151619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60bdaea7-2050-4e0d-bad5-0e5b6b1bcf54]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.153388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:17.186402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:17.187964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:17.188872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:17.190217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:17.190768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:17.262851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be33bdce-a102-4653-a530-c73da7e2a483]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cba7693e-1619-4fa1-99d6-bed4096f7720]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cba7693e-1619-4fa1-99d6-bed4096f7720]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:17.352444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:17.360010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[713edd29-da42-4aab-a64c-c225980bae53]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.366241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:17.366241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:17.366241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:17.366241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:17.366241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3325b2a-b265-4ec8-958d-7ac90431713b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.366241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35882059-da1b-43ad-930b-c86d1f44549b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.366241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd24d36e-956b-454f-946c-9758f6e25e21]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.387326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[927d40aa-b1de-4552-a8ce-4ea0c9dc24a2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 721328, 'reachable_time': 21080, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 214014, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.396386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:17.396965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04338841-c4e2-4bba-a9f5-3dedb7dc3102]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:17.398430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:14Z", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:17.404669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id 16e1a644-d2e0-4ad3-b35b-f349d80109ef. Wait time elapsed: 13.873 Jul 01 10:45:17.446034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id 16e1a644-d2e0-4ad3-b35b-f349d80109ef. Wait time elapsed: 13.873 Jul 01 10:45:17.446034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:18.143038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.742s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:18.143706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:18.144770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:18.145107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:18.145398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:18.147091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf3eade0-7830-4165-8433-442c01b01c21]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e41e544a-030b-4c85-b60e-bfd9f696311e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.180211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[612a571f-6f34-47b4-a192-5830b07660b2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.208843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36b02780-8913-454d-97fb-5415842773ad]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 214020, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.229589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4ec79a7-9eb0-44e4-842b-9d2f559370fa]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.250752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c9c8a376-bf53-4c94-ba1e-c44af9cc029f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.271875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcb2762b-9a23-4f5f-968c-6a11eb05b280]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.291872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15be0c01-536a-4fea-9938-8649a42b2b67]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.292418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:18.292755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:18.293121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:18.293458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:18.308554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:18.308849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f688b34-51ed-4a98-aff2-3c86ec441b0b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.311576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[916ce5c9-840c-4069-a58a-f3acebacfe6b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.312501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:18.344288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:18.344959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:18.345666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:18.345780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:18.346103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:18.395584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f98a6bff-7252-4afd-8555-c396421c9d3b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[13a0aa72-b680-4010-9828-bdbdcc4d8387]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:18.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:18.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:18.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:18.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:18.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:18.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:18.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:18.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:18.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:18.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:18.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:18.438036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:18.440024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13a0aa72-b680-4010-9828-bdbdcc4d8387]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:18.440965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:18.442973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:18.445484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fab51e24-1c40-4a2a-83ae-9de10ea01849]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.445484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:18.445484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:18.445484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:18.445484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:18.446998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27af6a1a-1bc4-4669-a18a-00e2f577a852]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.448235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9380270-f5ce-4fe5-ac99-e33ac4c70663]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.449146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ddf933e-3858-4966-8cb4-90d3e2c02bd0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.468082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff0b027b-1eed-44de-8f87-d786dfa4dd71]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 721447, 'reachable_time': 19620, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 214038, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.475568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:18.475926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7772b3bd-43a3-476a-8133-7a62dacae6d2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:18.477924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:16Z", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 13, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:18.481043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id f392b58f-0a12-48c1-8b37-17f75d881270. Wait time elapsed: 17.970 Jul 01 10:45:18.494485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id f392b58f-0a12-48c1-8b37-17f75d881270. Wait time elapsed: 17.970 Jul 01 10:45:18.494485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:20.946186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-411c60ed-ad4f-4fd3-81fa-065c017ed2f8 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['e4c7d557-c62d-4089-9e91-1907e88396ea'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:45:21.090808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.610s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:21.091424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:21.093068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:21.093419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:21.093695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:21.102395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27c4d8c3-93b5-4512-bc64-2515407d2d59]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.160050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97f4b6a0-e17b-47d1-a9ab-551fb77612b1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.229349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c675086-26a5-4ed3-9b78-55da27a73c25]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.317576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ccd0ebad-72c6-4a89-906e-c3ed7bc5faa2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 214076, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.367861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06adff64-7c78-4b24-b51b-e790b737e9ef]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.398402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46bcd3f1-ce95-4580-a841-8f68452a081c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.437887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83dd729f-e8e0-4b99-ae6e-537f64a90ba7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.477447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[949128e4-3f1a-41ef-9f7c-9437240b3b8c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.478078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:21.478413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:21.479141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:21.483292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:21.504680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:21.505228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7fb9f405-5227-4d35-bcdf-69735ff0c463]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.508885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5311de85-0c09-46c6-871f-b997c4427dc8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.509887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:21.557344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:21.558407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:21.559130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:21.559764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:21.560309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:21.624762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0bef14d-33b0-47d0-8f0e-8a318a0add60]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.687156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0cdc124b-3b2a-41c6-9b15-093a8dabef88]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:21.687156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:21.687156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:21.687156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:21.687156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:21.687156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:21.687156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:21.687156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:21.687156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:21.687156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:21.687156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:21.687156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:21.687156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:21.688793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0cdc124b-3b2a-41c6-9b15-093a8dabef88]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:21.690654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:21.694237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:21.699514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bb46e71-2cc0-4bef-855d-56f474ec246b]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.700606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:21.701768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:21.702188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:21.702687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:21.705101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[975cbcfd-88a9-4f78-91da-647ab25e0df6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.706612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50b6e394-15c6-4482-a7e8-699bf301a94f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.708110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f4d860e-0c06-4a60-bd19-f049db88f200]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.732723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d85541c-ca60-46c8-a5f0-1f03591a6361]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 721757, 'reachable_time': 37252, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 214106, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.742134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:21.742392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fce8f791-2a44-4fd4-a8cf-c586502c07d0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:21.743463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:19Z", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:21.747083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 1, update_id dd894b18-aa0f-4c5a-9d07-a0cfe64c7416. Wait time elapsed: 0.798 Jul 01 10:45:21.762696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 1, update_id dd894b18-aa0f-4c5a-9d07-a0cfe64c7416. Wait time elapsed: 0.798 Jul 01 10:45:21.762696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:22.621108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.876s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:22.621796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:22.622343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:22.622618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:22.622903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:22.624012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3aaa3c6-391e-489b-8b53-2e6bd92b5fd7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:22.647721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01c693d7-5619-4524-823e-1425c9a63f3d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:22.672068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf5bc9b5-f54c-4817-9c57-e5e3bdfcf624]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:22.716212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb361631-b4f9-4a02-8c73-95e23be48a35]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 214109, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:22.744351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c6fab0b-e53c-42ce-b255-b937b6707aae]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:22.768560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3117155-f8b5-4fb0-b917-c246d59ddc04]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:22.790787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[beb6617d-5ee2-4b19-99d6-407dbe473e47]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:22.844256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[217d1ab3-dda9-4a71-860b-b65df99899dc]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:22.845668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:22.846610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:22.855754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:22.856054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:22.873582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:22.873872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffadeefe-dc4f-4ad5-ac25-04b401008b3d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:22.877635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95a8f909-46d4-4f64-b1d1-92c9ac51857a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:22.879177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:22.937426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:22.950176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:22.951440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:22.956106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:22.958758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:22.997942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd8381db-597a-412e-8a2a-194ba4dacc00]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:23.035844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[86a7d451-4108-4f46-be86-4ec9d0e88c8f]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:23.035844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:23.035844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:23.035844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:23.035844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:23.035844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:23.035844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:23.035844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:23.035844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:23.035844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:23.035844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:23.035844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:23.035844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:23.038265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86a7d451-4108-4f46-be86-4ec9d0e88c8f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:23.039134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:23.042214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:23.048884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5011ec00-be59-43ba-bb76-5b3291f79350]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:23.050167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:23.050647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:23.051056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:23.052188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:23.053519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05a7ef02-0359-486d-8bff-39d363ecf57c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:23.054946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[204848af-c3f5-4857-ad80-3daa6bead9c5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:23.056077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5686c355-4d21-4892-bbdd-d3776704656c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:23.076997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[022deddd-6361-449b-88dd-5ce230ac50f4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 721897, 'reachable_time': 23496, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 214129, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0fdc534-7ac3-4ebe-b05b-0926f31e6ac9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:23.082304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1f412d79-5900-44c4-8126-e69d21f4539f", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.237", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "eed0734e-3a98-401d-818e-ccb946c15d49", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6162, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:b0:5e", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8a7a9529-7aaf-4f53-94b6-f9d0867a2ace", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:20Z", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:20Z", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:21Z", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:23.088382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id 16e1a644-d2e0-4ad3-b35b-f349d80109ef. Wait time elapsed: 19.555 Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id 16e1a644-d2e0-4ad3-b35b-f349d80109ef. Wait time elapsed: 19.555 Jul 01 10:45:23.102697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:25.894441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.812s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:25.895021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:25.895660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:25.895987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:25.896765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:25.898006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb0f3cf1-9436-40af-aca7-ffe8ceef0337]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:25.912990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7c4ed0d-290d-48ab-9109-322d6c4e076a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:25.937411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a2ed78d-500a-414f-ab6d-5f6f90f71f25]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:25.972666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31527794-64d2-4c4b-a32e-2bfe872101bb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 214135, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:25.996474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19e57353-c886-42f4-b4b1-cc4d571aa9d4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.021701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[199df24a-4e34-4a9f-a4c0-b1eb2893eb1a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.044425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a90cedae-29d2-4feb-ac46-f83c8f42dfeb]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.068358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17649928-02de-4f16-9da4-b32c0c5c3b56]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.068911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:26.069183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:26.069452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:26.069730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:26.085825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:26.086799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c53c46b7-535e-433a-91a2-dc59e76b3dc8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.090406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3ca6d0d-2995-4b06-8306-5390bf7b2506]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.091431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:26.128886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:26.129556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:26.130143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:26.130805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:26.131418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:26.200293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c582420-394e-4081-9a38-8dd8b447631f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.257412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a0ce01ae-28f7-4a49-8bd6-29887e9a9786]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:26.257412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:26.257412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:26.257412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:26.257412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:26.257412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:26.257412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:26.257412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:26.257412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:26.257412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:26.257412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:26.257412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:26.257412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:26.258625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0ce01ae-28f7-4a49-8bd6-29887e9a9786]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:26.260455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:26.263789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:26.269713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[765fa11a-f132-438a-ab3f-905dccf23c73]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.271258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:26.271749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:26.272069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:26.272774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:26.277246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88e357ee-350c-433d-9a20-590f75159712]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.281100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb9e8c05-b4bd-485d-a124-06f70dccde79]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.282882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61e4fd9e-f246-4206-bacc-84b3fcb0a541]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.318155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7cb4816-46c6-4e23-84fe-ecc63db8470f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 722223, 'reachable_time': 40720, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 214164, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.325209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:26.325472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a5d4fb0-40a0-476d-ad19-58b75558fede]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:26.326506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1f412d79-5900-44c4-8126-e69d21f4539f", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.237", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "eed0734e-3a98-401d-818e-ccb946c15d49", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6162, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:b0:5e", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8a7a9529-7aaf-4f53-94b6-f9d0867a2ace", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:20Z", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:20Z", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:26.330487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:23Z", Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id f392b58f-0a12-48c1-8b37-17f75d881270. Wait time elapsed: 25.817 Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id f392b58f-0a12-48c1-8b37-17f75d881270. Wait time elapsed: 25.818 Jul 01 10:45:26.344727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:27.194974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.867s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:27.195611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:27.196235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:27.196557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:27.196838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:27.198705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[febdcca7-b3ab-4881-bae7-d2d723783257]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.216518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5b5ee27-b0ff-41e5-9ccf-424207ba9078]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.242622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2200df5f-910c-4f63-95e3-c47708db54d2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.287175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29b61c43-1a32-4338-b279-ac46ac0839cd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 214218, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.311697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8550ac3-7e6a-43b4-8094-93d394299a13]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.343789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e75d802d-3fe4-4a56-a7b8-8cb678eb3a9c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.366939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfab7f59-5804-4bff-b1a5-318738f430dc]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.389626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8b46d50-0036-48a0-9e62-3bdf228e40ab]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.390429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:27.390974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:27.392090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:27.392605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:27.413348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:27.413722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[801904f0-90ee-488d-8780-fd583eff49ca]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.418877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b7a79d9-c848-471f-9dbd-bed0438b88d3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.420404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:27.458164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:27.459147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:27.459578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:27.459968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:27.460346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:27.518030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d449255-9bb1-4d2a-b7d9-0f1f5371c0fe]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.559257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3c8b320a-4d56-4afa-9987-c7e766382d09]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:27.559257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:27.559257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:27.559257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:27.559257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:27.559257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:27.559257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:27.559257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:27.559257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:27.559257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:27.559257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:27.559257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:27.559257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:27.561293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c8b320a-4d56-4afa-9987-c7e766382d09]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:27.562301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:27.565044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:27.567821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b533f5c0-dbc7-4780-a194-6aa2d4ad91d2]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.567821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:27.567821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:27.567821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:27.567821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:27.569018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4bb80a9-abfb-45a9-b9d6-865064427255]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.570562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26f08f2e-f602-494a-84ac-a44307ffc35f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.571672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f9a7928-84b8-4965-8023-dbc4a0f25463]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.592059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[078ce7c1-f2e4-47f5-afdc-d52919caec01]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 722354, 'reachable_time': 19444, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 214259, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.600530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:27.601034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c0e6f85-3e30-4027-a701-f6801f338fe5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:27.602728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1f412d79-5900-44c4-8126-e69d21f4539f", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.237", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "eed0734e-3a98-401d-818e-ccb946c15d49", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6162, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:b0:5e", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8a7a9529-7aaf-4f53-94b6-f9d0867a2ace", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:20Z", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:20Z", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:27.608529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:25Z", Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:27.625518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:27.631857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3 Jul 01 10:45:27.631857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id dd894b18-aa0f-4c5a-9d07-a0cfe64c7416. Wait time elapsed: 6.681 Jul 01 10:45:27.631857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id dd894b18-aa0f-4c5a-9d07-a0cfe64c7416. Wait time elapsed: 6.682 Jul 01 10:45:27.631857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:30.026260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.398s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:30.026904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:30.027643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:30.028045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:30.028373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:30.029584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dfb959af-9a6a-43c5-8b1e-2f7d15af8e1d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.044178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f14ec1a0-3b93-43ba-88fa-c32e4b068b8e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.066635 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb93d98c-0102-477a-b2d0-10f0c1a5b284]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.099333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40abf453-9b18-485b-aafc-cedfa206f020]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 214266, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.128581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f71b5ed-d77c-4fe1-ab58-4eecfe5d50ce]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.154806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d458cf8-4160-4bae-90ac-cf819ef904ab]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.176996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[225fbe9e-9369-4122-90d8-acdc753bca25]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.198406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44851a35-cdf0-4830-916e-2e2e3d717bd1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.198976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:30.199286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:30.199604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:30.199889 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:30.216595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:30.216892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1e283ef-1aae-46b3-924c-77d7207dc55a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.219585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7465f045-7497-40c9-9799-ff42e0e62439]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.220522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:30.258254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:30.259238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:30.259676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:30.260102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:30.260639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:30.320378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[299440f8-4cf6-47c1-9fc6-aab52e95b96a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.350936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[65374caf-1f76-4fcf-84ac-9532fa05b8e9]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:30.350936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:30.350936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:30.350936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:30.350936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:30.350936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:30.350936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:30.350936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:30.350936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:30.350936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:30.350936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:30.350936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:30.350936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65374caf-1f76-4fcf-84ac-9532fa05b8e9]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:30.352244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:30.354439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:30.357660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[39739658-ac9f-4173-825b-f2753f1a1c35]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.357660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:30.357660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:30.357660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:30.357660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:30.357660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[802de3e9-8fd0-43a5-b70e-9f3f05a73bee]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.358812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35f01633-adfd-49ea-a396-550b7e9bf3ca]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.359658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc56aed1-8552-4bb4-8825-6d6ccc66b14c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.382115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62d7a74b-e848-4f3d-93d0-8188c866b241]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 722636, 'reachable_time': 16352, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 214286, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.384170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:30.384598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e66b67be-657b-4c0f-bdd5-626b73845fe1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:30.385730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1f412d79-5900-44c4-8126-e69d21f4539f", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.237", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "eed0734e-3a98-401d-818e-ccb946c15d49", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6162, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:b0:5e", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8a7a9529-7aaf-4f53-94b6-f9d0867a2ace", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:20Z", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:20Z", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:30.391060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:28Z", Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id 16e1a644-d2e0-4ad3-b35b-f349d80109ef. Wait time elapsed: 26.865 Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id 16e1a644-d2e0-4ad3-b35b-f349d80109ef. Wait time elapsed: 26.865 Jul 01 10:45:30.407279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:31.372757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.981s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:31.373382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:31.374154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:31.374535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:31.374878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:31.376652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fda32f55-2999-45cb-a05e-12284c300e04]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.390786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac856792-386a-4ed1-968d-74504b5e1cec]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.412233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ecd17ec-65a1-4d10-a02d-ef56f73b614a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.440941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d76e4bf-d81e-47a5-b2dc-cd5379af8b23]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 214334, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.461874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f85acee-7ef1-4993-b7bb-a311f3112896]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.483099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71a004f0-5c73-4f94-8001-06379d09e686]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.504775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[628276b2-619d-41f7-882a-58938330b683]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.527223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67169453-280d-409a-b52d-8527433048dd]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.527974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:31.528321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:31.528663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:31.529013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:31.546324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:31.547019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3dfdcee-c0c9-457d-a5bd-d9ef53f449c3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.550585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21e683ea-fe8c-48f2-ad6b-841516d93756]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.552278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:31.587070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:31.588100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:31.588785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:31.589400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:31.589853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:31.632457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17ea89f8-118c-4ea0-abf1-48ca56d5bbdd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.671872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[877e82cb-5770-4c1a-be03-3bb8feb4b10d]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:31.671872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:31.671872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:31.671872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:31.671872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:31.671872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:31.671872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:31.671872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:31.671872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:31.671872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:31.671872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:31.671872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:31.671872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:31.673080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[877e82cb-5770-4c1a-be03-3bb8feb4b10d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:31.673225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:31.675379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:31.678096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40af9fcc-48fa-4e1a-ba57-fcc34510a6d2]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.678096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:31.678096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:31.678096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:31.678096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:31.678096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0929bd39-9006-41b7-8035-b13bca330449]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.678984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2084ce93-0a22-4628-88fe-ba336a0bd3b4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.679961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf6af913-3594-46d3-8b32-61a9de5be651]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.699819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1801fca-077c-43e1-9c68-995d3249df62]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 722770, 'reachable_time': 23380, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 214352, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.702092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:31.702516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1d8f30a-6081-4cfd-8daf-5afb4c0601dd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:31.704603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1f412d79-5900-44c4-8126-e69d21f4539f", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.237", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "eed0734e-3a98-401d-818e-ccb946c15d49", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6162, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:b0:5e", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8a7a9529-7aaf-4f53-94b6-f9d0867a2ace", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:20Z", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:20Z", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:31.709343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:29Z", Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 21, Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3 Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id dd894b18-aa0f-4c5a-9d07-a0cfe64c7416. Wait time elapsed: 10.763 Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id dd894b18-aa0f-4c5a-9d07-a0cfe64c7416. Wait time elapsed: 10.763 Jul 01 10:45:31.727090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:33.960084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.250s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:33.960616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:33.961246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:33.961616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:33.961893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:33.963030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5b7d997-6740-4a89-a8c6-07534b74da4c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:33.976727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8fdbe26c-8fbd-4150-843d-7f8800be5095]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:33.998685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da613563-5b8e-4aad-94bd-66955c264d73]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.027763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4eb58f70-d7fa-4e36-8657-a28a46cfd522]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 214357, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.050647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06ee3a75-f9f3-4f1c-872d-d41b6b4f9e63]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.071818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12217fdc-8efb-44d7-8cc8-7aaee79e847a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.083959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ac2298b-6b18-45e8-8e60-2493940d038b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.107755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b54ef63-9d87-4bf5-952f-ff91d6e79733]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.108309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:34.108590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:34.108860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:34.109141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:34.127498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:34.128561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8404025-2e92-4fb0-8cc6-a276b03afe84]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.132730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5b9d071-511e-45cc-8cc2-97edcb6a430b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.134370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:34.171904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:34.173557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:34.174003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:34.174566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:34.175016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:34.223931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a5999b1-638a-4db7-9d1b-cdc8a1ceee9a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.267849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ba791ce2-4719-4861-b363-4bde586744ee]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:34.267849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:34.267849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:34.267849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:34.267849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:34.267849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:34.267849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:34.267849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:34.267849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:34.267849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:34.267849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:34.267849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:34.267849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:34.268924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba791ce2-4719-4861-b363-4bde586744ee]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:34.269574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:34.272872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:34.275652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b88c174-ebcc-4434-a7a0-538df3b9ae2f]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.275652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:34.275652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:34.275652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:34.276027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:34.288183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d1d7ecd-0ae3-495c-a421-17df1b139dad]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.289307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cdd95789-ec14-48cf-a5c1-190d833cf9b8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.290276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ba63e4d-7dd4-4b8f-bfe8-1a97bfaf9819]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.311412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[729e376b-b378-41c7-9013-637c6e1ba577]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 723029, 'reachable_time': 29456, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 214377, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.317619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:34.318004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[683c7cf8-ca15-4086-91d7-bce03a097b1a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:34.319392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1f412d79-5900-44c4-8126-e69d21f4539f", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.237", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "eed0734e-3a98-401d-818e-ccb946c15d49", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6162, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:b0:5e", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8a7a9529-7aaf-4f53-94b6-f9d0867a2ace", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:20Z", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:20Z", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:34.322737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:32Z", Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 22, Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id dd894b18-aa0f-4c5a-9d07-a0cfe64c7416. Wait time elapsed: 13.374 Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id dd894b18-aa0f-4c5a-9d07-a0cfe64c7416. Wait time elapsed: 13.374 Jul 01 10:45:34.340364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:35.118701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.797s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:35.119295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:35.120094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:35.120489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:35.120855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:35.122063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d71cd94-0e6a-4e43-9250-682b0ffc791c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.137298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb883e45-3d0d-4558-a2da-ec8859745f37]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.158688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0aa15be-72ad-4a8a-a667-161ddf18f853]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.191332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef97c827-5b30-4d79-b45d-8f26714490d2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 214380, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.212247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75107235-7509-49dc-ae8c-9a2ea3718a58]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.233328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e906f4c-b9ce-41f6-ae40-315dc26a856a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.253878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[324b56d8-d7a2-4e0d-9a52-a4ddaedb25e4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.274474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ede9b784-d845-4bf6-b8bd-ed780f8b4a21]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.275054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:35.275409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:35.275741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:35.276079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:35.292327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:35.292628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50719370-c6ac-4bf8-b10e-0ef994075663]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.295323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[691ba7b3-77b8-401a-8578-3632b10488b7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.296337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:35.321538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:45:35.335706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:35.336486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:35.336947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:35.337693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:35.338173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:35.407687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[384c932a-9178-48dd-8adb-6c11a55a5823]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.454978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f50fc5bb-4d6a-4e14-b41e-10d57c10a616]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:35.454978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:35.454978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:35.454978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:35.454978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:35.454978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:35.454978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:35.454978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:35.454978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:35.454978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:35.454978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:35.454978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:35.454978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:35.456550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f50fc5bb-4d6a-4e14-b41e-10d57c10a616]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:35.457882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:35.461026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:35.465262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3322656-f9b9-4db4-9db1-05894a1ec002]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.466466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:35.467255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:35.468163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:35.468754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:35.470008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf1f9185-9d3a-4867-be6a-fc37a5198922]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.471063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[741938fa-213f-401e-b07a-089dadcb6bf4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.472400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b17efd13-36f1-4242-b8ee-8071c373403c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.496674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9997681c-bc17-41ce-9c55-98ebefe6e69d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 723145, 'reachable_time': 17864, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 214403, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.504833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:35.505335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d68f0b9-1c53-431a-8d98-ceb4f6aec0fd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:35.506362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1f412d79-5900-44c4-8126-e69d21f4539f", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.237", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "eed0734e-3a98-401d-818e-ccb946c15d49", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6162, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:b0:5e", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8a7a9529-7aaf-4f53-94b6-f9d0867a2ace", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:20Z", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:20Z", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:35.510579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:33Z", Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id dd894b18-aa0f-4c5a-9d07-a0cfe64c7416. Wait time elapsed: 14.561 Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id dd894b18-aa0f-4c5a-9d07-a0cfe64c7416. Wait time elapsed: 14.561 Jul 01 10:45:35.528032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:37.941139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.433s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:37.941762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:37.942305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:37.942626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:37.942898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:37.944061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc1e6d16-6f97-4a3f-bcc7-21b3346f1d59]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:37.957393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f1796d8-5e55-448e-8c21-317dca1acce7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:37.977598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c153e89-a8ec-4458-a96b-bdfad27ca02b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.008877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac275123-a4d1-43f3-84de-0c79cd8b1830]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 214452, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.030411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[284cc060-a940-429b-9c6b-5259ebc3b0bc]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.051614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8823616-9428-45f6-91d0-a67a22eb16f0]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.071857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41af2854-3663-4db5-9b2f-10f2cce66999]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.092997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30255b00-cc04-4479-801a-2c67f9649760]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.093638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:38.093915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:38.094265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:38.095327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:38.113549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:38.113982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5f8d32d-ae0e-4a3a-bbd2-7aeb075d3774]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.117364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2c1f862-0fc0-4708-84e3-f748c1a07065]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.127219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:38.166326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:38.167504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:38.167907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:38.168296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:38.168681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:38.230591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36d3eec7-98a1-451d-bbcf-43fd072fcd37]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.260456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2914d0d3-de43-4518-ba67-0d7ce6b48866]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:38.260456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:38.260456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:38.260456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:38.260456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:38.260456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:38.260456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:38.260456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:38.260456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:38.260456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:38.260456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:38.260456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:38.260456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:38.262265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2914d0d3-de43-4518-ba67-0d7ce6b48866]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:38.263534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:38.267449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:38.272594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74ccad68-41f5-41ee-b5a8-f1ba90b1740a]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.273959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:38.274482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:38.275014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:38.275563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:38.276851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd2fdd99-4dbc-4c10-989d-b937cf2cbc7a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.277984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f3f2a5f-0d2b-4e22-b041-91a762b583c0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.279561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83aa5213-c20c-423b-8933-b367b3126813]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.300715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54db612d-47f0-4f64-b0e8-aebea7207cc8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 723427, 'reachable_time': 22600, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 214470, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.307619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:38.307919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60ac6066-1e6d-41e9-bc0f-902aba1022b4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:38.308937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1f412d79-5900-44c4-8126-e69d21f4539f", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.237", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "eed0734e-3a98-401d-818e-ccb946c15d49", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6162, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:b0:5e", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8a7a9529-7aaf-4f53-94b6-f9d0867a2ace", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:20Z", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:20Z", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:38.312527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:36Z", Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 24, Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id dd894b18-aa0f-4c5a-9d07-a0cfe64c7416. Wait time elapsed: 17.364 Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id dd894b18-aa0f-4c5a-9d07-a0cfe64c7416. Wait time elapsed: 17.364 Jul 01 10:45:38.333982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:45:39.213483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.903s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:45:39.214104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:45:39.217563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:45:39.217893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:39.218176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:39.220072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[64a61cda-d85e-4a87-80bf-863f50b4f3d8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.232034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[79772b9f-bcfa-49c2-8c53-f46b4831823d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.255532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9b2244b-2913-465c-b522-613dc68211b7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.290636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7541096c-2b1c-4b5b-b733-7237d053479f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 214473, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.311784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[747d0dcb-cd4b-45d4-815a-fbc9a56210b1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.335317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bc0a71f-1072-4dfe-8aa4-3d292c05728d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.359137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a71fa49-6ecf-477a-a1e9-e3ec3d5aa877]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.380593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bd81472-9caa-4185-a082-a01e609353a9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.381166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:45:39.381451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:45:39.381859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:45:39.382309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:45:39.401033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:45:39.401890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0930a71a-4f73-49a1-836e-5fb95a0da731]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.404658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3b31fc9-a384-4c19-a909-da535e1aee9c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.405607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:39.443108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:39.444071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:39.444665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:39.445312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:39.445766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:45:39.519282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7608e4d3-df7d-42c0-8cf7-44a75a13f6e0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.550976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7166116c-5f7e-4c3f-ae32-9734aa503933]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:45:39.550976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:45:39.550976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:45:39.550976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:45:39.550976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:45:39.550976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:45:39.550976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:45:39.550976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:45:39.550976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:45:39.550976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:45:39.550976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:45:39.550976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:45:39.550976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:39.552075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7166116c-5f7e-4c3f-ae32-9734aa503933]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:39.552180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:39.554002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:39.556421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9977c285-181f-4205-be7e-681deb568668]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.556421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:39.556421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:39.556421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:45:39.556421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:45:39.558115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e4c5cc3-3e82-4263-be82-d02b4554b826]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.559174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[036d5143-391a-408b-9fcd-12112b11a330]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.560192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74e1a403-5e09-4556-aee1-b326866bd2ec]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.580982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b884445f-2a38-4cc4-8af4-faa9612e9bf4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 723555, 'reachable_time': 44380, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 214492, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.587510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:45:39.587766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e48ce2c-7749-4702-9abf-42b5082a2547]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:45:39.588851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "1f412d79-5900-44c4-8126-e69d21f4539f", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.237", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "eed0734e-3a98-401d-818e-ccb946c15d49", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.6", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6162, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a4:b0:5e", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "8a7a9529-7aaf-4f53-94b6-f9d0867a2ace", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:20Z", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:20Z", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:45:39.591994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:37Z", Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 25, Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:45:39.607119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3 Jul 01 10:46:15.159747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:46:15.160461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:46:15.160461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:46:15.160669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:46:15.322528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:46:55.326593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:47:15.160402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:47:15.161032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:47:15.161032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:47:15.161157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:47:35.328434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:48:15.160622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:48:15.161322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:48:15.161322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:48:15.161476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:48:15.329489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:48:55.330786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:49:15.161610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:49:15.162503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:49:15.163084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:49:15.163576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:49:35.332405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:50:15.162254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:15.162937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:15.162937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:15.163119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:15.334004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:50:22.730501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-5db2bb68-090a-4b9f-a10d-d070d2031ed5 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['e4c7d557-c62d-4089-9e91-1907e88396ea'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:50:22.731749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 1, update_id e0c948bb-f6ad-4e82-9d7d-1ad93cf1eef9. Wait time elapsed: 0.001 Jul 01 10:50:22.732236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 1, update_id e0c948bb-f6ad-4e82-9d7d-1ad93cf1eef9. Wait time elapsed: 0.002 Jul 01 10:50:22.732664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:24.788551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.056s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:24.789084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:24.789756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:24.790194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:24.790484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:24.791666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e7138b6-7a50-4d94-b3c3-18777e1bae5f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:24.805584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27f551f0-8042-4a48-8726-0a90769e11bc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:24.828534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[176e712e-0012-4c57-8329-f2225d614b8d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:24.868283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6611d8ee-6772-4005-960e-eea235f2616f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 218968, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:24.890707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c7bc657-7aea-4318-b17a-48a2561d25bd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:24.933942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa5028c4-18fc-461e-bd77-05fbf40abb32]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:24.963402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65d48e2e-331a-4240-9745-3ec30257bfd8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:24.986981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e45e4e25-166c-423a-8f4a-d42df1e87c8b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:24.987802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:24.988089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:24.988353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:24.988680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:50:25.006809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:25.007203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d952b35-bf29-4ecb-82eb-2a624ae23165]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.010023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa341343-73d9-48a9-8339-03123f54b77b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.011694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:25.048310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:25.049048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:25.049448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:25.049834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:25.050218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:25.095864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[251e0335-0d85-4dff-a961-9116977620b5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.126639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[495683a0-8e91-4eb0-8157-16bd7912e9bb]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:25.126639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:25.126639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:25.126639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:25.126639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:25.126639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:25.126639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:25.126639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:25.126639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:25.126639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:25.126639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:25.126639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:25.126639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:25.127584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[495683a0-8e91-4eb0-8157-16bd7912e9bb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:25.128065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:25.129733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:25.131810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[833b9d94-19c4-44ff-8f68-1daf451e7bea]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.131810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:25.131810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:25.131810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:25.132120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:25.133108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[983e3e4e-64be-4199-ab01-00ff386a2190]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.134094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9d8cd18-34a4-486f-9866-ffb1f0202a7e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.135090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[742df4e4-ce4c-4182-980d-b6fafb5c1cb1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.159295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f733d127-1e8e-4417-875a-29d0a4ae04e4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 752112, 'reachable_time': 18292, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 218986, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.164675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:25.164675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5f6dab7-45e2-47e5-a735-4da9a4bfecc1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:25.165349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:41Z", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 27, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:25.167794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id e0c948bb-f6ad-4e82-9d7d-1ad93cf1eef9. Wait time elapsed: 2.436 Jul 01 10:50:25.179548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id e0c948bb-f6ad-4e82-9d7d-1ad93cf1eef9. Wait time elapsed: 2.436 Jul 01 10:50:25.179548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:25.863242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.696s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:25.863899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:25.865689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:25.866023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:25.866313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:25.868059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b046895-54ce-45b6-9d46-1f132f98b7c0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.882764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ba73b22-2dbf-41c4-9176-05c67215d192]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.907980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eafbf28b-3a2e-47be-867c-2b3a793d1c93]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.937130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df1a444b-babc-486b-8e89-ea4a743ee18c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 218989, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.961817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78d81b2d-946d-45a5-9fdf-3a7eda544d93]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:25.984630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b90fcf6b-a862-4a9e-9f1d-4f7d8ea49816]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:26.007845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2dfa046f-dddd-44f9-b293-729075255cd7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:26.030087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb7ecb20-b5f5-462c-bc2f-1b2669d10a59]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:26.030706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:26.030984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:26.031250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:26.031562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:50:26.049940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:26.050145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac87dda2-2b46-4dab-8abb-f5cf29c2384d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:26.053548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e63ea5ff-07c3-4143-b18d-ba09edf145f7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:26.054954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-dd3f0214-f4, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:26.084342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-dd3f0214-f4, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:26.085104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:26.085531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:26.085930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-dd3f0214-f4, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:26.086320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-dd3f0214-f4, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:bc:cd:71'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:26.150109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbfa03af-280e-4ac8-b059-6250e4ac8da7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:26.190754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[196ecedd-28bf-432e-b4e9-79d04ffa4c95]: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:26.190754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:26.190754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:26.190754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:26.190754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:26.190754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:26.190754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:26.190754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:26.190754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:26.190754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:26.190754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:26.190754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:26.190754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:26.191926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[196ecedd-28bf-432e-b4e9-79d04ffa4c95]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:26.192004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:26.193414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:26.195396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff76d793-cbc5-4dfe-87aa-7533ca12c4bb]: (4, ['qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:26.195396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:26.195396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:26.195396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:26.195722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:26.196816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0a60e5c-10d7-42c3-af22-1381e478a0e9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:26.198043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc881069-a8a9-4d8c-b1ac-84ae67f0c4da]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:26.199393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef729c12-52cc-4b42-a2cd-bc0f3589e4d0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:26.226257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d534221e-d1ba-4f2c-82b7-8de3a24eb9ce]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 752220, 'reachable_time': 27748, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219007, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:26.235998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:26.236128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf90e501-bab6-472e-9c4a-c764336ccbf3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: e4c7d557-c62d-4089-9e91-1907e88396ea: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-dd3f0214-f4 not found in namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea. Jul 01 10:50:26.237218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-2034665815", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172" Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "afa5b574-8e50-499f-90e7-c4802a8ff425", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:d5:17:de", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6143, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.172", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:44:59Z", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:02Z", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd3f0214-f4bc-4bee-bc45-be2d0f2ab25e", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5eb98f20-b481-457f-b674-074e10bd84aa", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bc:cd:71", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "e4c7d557-c62d-4089-9e91-1907e88396ea", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6147, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:45:01Z", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:45:41Z", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 27, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "890a5746-36c6-49c1-bf83-8ebf35aabdd8", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:26.239660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id e0c948bb-f6ad-4e82-9d7d-1ad93cf1eef9. Wait time elapsed: 3.508 Jul 01 10:50:26.250903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 2, update_id e0c948bb-f6ad-4e82-9d7d-1ad93cf1eef9. Wait time elapsed: 3.508 Jul 01 10:50:26.250903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:27.132157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-82379a4b-52d5-448e-b361-69bf5477270f tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['e4c7d557-c62d-4089-9e91-1907e88396ea'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:50:28.143625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.905s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:28.144246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router e4c7d557-c62d-4089-9e91-1907e88396ea info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:28.145406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:28.145800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:28.146150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:28.148387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ef93615-c463-45ae-8d38-3d6c11f1260c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.171142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d081b0d-468c-4bc7-b488-0c9ad4e622c8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.195004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[762a2f5a-d1c1-484d-870e-cdb53d1257cf]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.235447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09e1b1bf-07cd-49a0-9c3d-e374b1271428]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219015, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.274769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac749a67-fecf-464d-aaab-58c435eafc34]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.302284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be939535-bd75-46d3-9deb-2922df76690d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.343218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b79e387-e82d-4a9f-bc1e-ae89fd92b0c7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.379987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[455a326a-3291-4f5d-b149-f23914debb28]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.380695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:28.381125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:28.381485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:28.417970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b2d6a86-4813-4a1a-ab9c-e487467b570c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 752449, 'reachable_time': 27756, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219029, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.419682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:28.420265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:28.422612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:50:28.423231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" acquired by "process_external" :: waited 0.002s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:50:28.476216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7b7ba78-45d4-4f49-a6c0-16dfbe49f24d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 752449, 'reachable_time': 27756, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219033, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.477938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:50:28.478686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:50:28.541247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6275242-ff73-444d-b4bf-627ea17417f5]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.761461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ca96d9d6-6dd6-4e7c-b14c-0a2d790dafa0 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got router deleted notification for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:50:28.780216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9446338e-966f-4144-90c7-65e28610c5aa]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.794290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ae7b5a0-eafc-4437-a3f4-b797299f6d81]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.971190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9be9ec7f-55e6-423f-b1f1-5e7ffe03245f]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:28.971719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:50:28.972605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:50:28.973051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:50:28.973491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" released by "process_external" :: held 0.550s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:50:28.974089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:50:28.974495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:50:28.974896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:50:28.975485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:50:29.022521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5ae6ee6-49df-40f0-8736-121a9f533876]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.082413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbfd8999-f5f8-4388-a328-4d0f3f6e4138]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.109896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d695379-2333-4e54-94ee-6bf5bc67cc94]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.177746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7881983-c07d-4284-857f-ecf86037cdfd]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.179827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:50:29.180568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:50:29.181268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:50:29.181795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" released by "process_address_scope" :: held 0.207s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:50:29.182297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:29.182827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:50:29.183166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:50:29.184748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:50:29.185609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:50:29.218184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[451af7f7-b18a-49df-9b18-6e08fd95a276]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.302695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df61813f-8c4f-4ca1-92f8-b61fed103819]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.321081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81788607-4311-445c-bed0-71957e83f02e]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 10:50:29 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 10:50:29 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.379393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56f246cc-d71a-4c0b-8d36-c5ab634ed876]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.380110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:50:29.380603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:50:29.380878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:50:29.381161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea" released by "apply_metadata_nat_rules" :: held 0.198s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:50:29.381654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-e4c7d557-c62d-4089-9e91-1907e88396ea Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.haproxy Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID e4c7d557-c62d-4089-9e91-1907e88396ea Jul 01 10:50:29.382186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 10:50:29.384195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/e4c7d557-c62d-4089-9e91-1907e88396ea.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 10:50:29.447822 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-e4c7d557-c62d-4089-9e91-1907e88396ea[219102]: Proxy listener started. Jul 01 10:50:29.453481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router e4c7d557-c62d-4089-9e91-1907e88396ea add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 10:50:29.453645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for e4c7d557-c62d-4089-9e91-1907e88396ea, update_id e0c948bb-f6ad-4e82-9d7d-1ad93cf1eef9. Time elapsed: 3.215 Jul 01 10:50:29.453917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 1, update_id 5f328bc2-a132-44bf-8477-ee5bc3e4ebc3. Wait time elapsed: 2.321 Jul 01 10:50:29.454157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for e4c7d557-c62d-4089-9e91-1907e88396ea, action 3, priority 1, update_id 5f328bc2-a132-44bf-8477-ee5bc3e4ebc3. Wait time elapsed: 2.321 Jul 01 10:50:29.454470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:29.492992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.038s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:29.494029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router e4c7d557-c62d-4089-9e91-1907e88396ea delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 10:50:29.494351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:29.519245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e24216b-2f81-433e-a3b2-56e723f7664b]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.521593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[196cedf3-270e-4b95-967e-79e585a9bb55]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.523648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73564967-9660-47a1-99f1-1f095000bcbb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.524594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 10:50:29.525630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b639abf-d56c-493c-8b65-f4bf5821c953]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.545995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96df413e-705c-43dd-b235-fae11035388e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 752449, 'reachable_time': 27756, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219108, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.547211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:29.547741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:29.575742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bbc604c-0fd7-4786-856e-5cafd34e1fcf]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 752449, 'reachable_time': 27756, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219109, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.576823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 10:50:29.577540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:29.578107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/e4c7d557-c62d-4089-9e91-1907e88396ea.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:29.578538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:29.580250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router e4c7d557-c62d-4089-9e91-1907e88396ea {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 10:50:29.581523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00492f4a-0212-4236-a7f3-b627f13bc60e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.606208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b9f6381-11e0-4faa-96e7-7af6eb1465db]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 752449, 'reachable_time': 27756, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219110, 'error': None, 'target': 'qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.613645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-e4c7d557-c62d-4089-9e91-1907e88396ea deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:29.614030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f53c4815-b2ee-4c40-9d87-1559749780a7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:29.614468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info e4c7d557-c62d-4089-9e91-1907e88396ea delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 10:50:29.614823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (e4c7d557-c62d-4089-9e91-1907e88396ea), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:29.615289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:29.615762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:29.616073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:29.616326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:29.616622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for e4c7d557-c62d-4089-9e91-1907e88396ea, update_id 5f328bc2-a132-44bf-8477-ee5bc3e4ebc3. Time elapsed: 0.163 Jul 01 10:50:32.525866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ed0973ec-89db-449c-aed8-0c7a8d77d1ac tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['0fb3819d-deb2-4bcf-ac9d-627e82b9fe54'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:50:32.526850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 1, update_id d2f2a74d-523c-4089-a17f-c126d536b88e. Wait time elapsed: 0.001 Jul 01 10:50:32.527342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 1, update_id d2f2a74d-523c-4089-a17f-c126d536b88e. Wait time elapsed: 0.001 Jul 01 10:50:32.527732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:35.162096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.634s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:35.162688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:35.163467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:35.163903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:35.164281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:35.165735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ce19873-0542-4d0e-b7f4-b7e08efb1837]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.182177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08e1ca63-9a86-491a-9067-900a17ff336c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.205631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e10c947-5348-45d5-8fe2-6a8fd8569c0c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.237955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[104ad879-65ce-4931-80d9-1577bf2b47e8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219176, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.259820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86efc223-50bc-4794-abe8-4596d3f386fb]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.282273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f739c08a-efd4-408a-ae8b-127061e9d823]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.306002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f336ea48-9f01-4151-9a22-3b898ba6bcc5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.332556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[214fccc3-d952-4b7a-b241-72786c956135]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.333699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:35.334011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:35.334326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:35.356562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10679c5d-a199-4954-a2e4-0e982aea56ff]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 753150, 'reachable_time': 44480, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219182, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.358674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:35.358957 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:35.359400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:50:35.359660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:50:35.359990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:35.360242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:35.360744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'b02c4734-0a9b-43bd-9c32-a0ff9aea7f09', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:a5:ec:88', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'device_owner': 'network:router_gateway', 'standard_attr_id': 6465, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.20', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:50:31Z', 'updated_at': '2022-07-01T10:50:34Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-b02c4734-0a), ns(qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:50:35.380117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:35.380643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6685f21d-a341-4a45-8680-8c8448eb37b5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.384928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abd850c0-f6b4-4cb4-8ea4-92008ec1dbd6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.386007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-b02c4734-0a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:35.386439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:50:35.387385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-b02c4734-0a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:35.388340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-b02c4734-0a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:35.388830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-b02c4734-0a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:35.389279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-b02c4734-0a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:35.389743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-b02c4734-0a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b02c4734-0a9b-43bd-9c32-a0ff9aea7f09', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a5:ec:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:35.445470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff67a06a-40fa-4dd5-ab41-3bf4335d3efd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.484080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[38ed01cb-8d2d-40ce-827f-50806624349b]: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:35.484080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:35.484080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:35.484080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:35.484080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:35.484080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:35.484080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:35.484080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:35.484080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:35.484080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:35.484080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:35.484080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:35.484080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:35.485199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38ed01cb-8d2d-40ce-827f-50806624349b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.485199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:50:35.485709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:50:35.499568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a40169c2-75c6-47a1-9881-5b6972006c33]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.513798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8919bfcb-6d03-4646-9cc1-e94b9bd6443d tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['0fb3819d-deb2-4bcf-ac9d-627e82b9fe54'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:50:35.671139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91fc5db2-1209-4de1-bbbf-b629c461efc1]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.683116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4e9e178-76a0-4a8d-a931-d53b28308b9a]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.835948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b22f20d9-2faf-4152-9334-00a237bed340]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.837013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:50:35.837339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:50:35.837672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:50:35.838026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" released by "process_external" :: held 0.478s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:35.838830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:35.842557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:35.849400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c9cf3f2-656f-4194-94f0-7ed940aa6afb]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.850616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:35.850616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:35.851933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:35.852417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:35.853738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[864a87a5-97de-4ae2-bf9f-1d32bc46f42b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.854786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07b9141e-8f90-4d3d-a0cc-6d9c4906c70a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.856965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbdee3a5-0017-451a-8db1-92681f10a2e7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.877549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb38d55e-fc4b-4e53-9dc2-fee3f49485e5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 753150, 'reachable_time': 44480, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219199, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.884518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:35.885022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[045f6df3-6abb-4995-a343-e9caf89c2c4d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:35.886198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:31Z", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:50:35.891081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:35.898575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 1, update_id fb997476-0c94-49c8-acdc-c1c770fa63fa. Wait time elapsed: 0.377 Jul 01 10:50:35.898575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 1, update_id fb997476-0c94-49c8-acdc-c1c770fa63fa. Wait time elapsed: 0.377 Jul 01 10:50:35.898575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:37.713353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.821s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:37.714272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:37.716217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:37.716846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:37.718012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:37.720427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fae48cd4-d385-4243-a0e4-550c09bec90f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:37.735843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a028bcbe-81e0-43aa-8db7-624a1c2efcdb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:37.767504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e84801ee-e91d-40ad-bd7f-2cb15a3e7c24]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:37.812153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[855b55f5-5015-4dd5-a4e5-eb8d6d1b087c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219254, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:37.835254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4f72390-fd27-4859-84d2-8da661d2f906]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:37.862848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[545800bc-6bd7-43e6-b724-b72e80498880]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:37.889735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc626cb0-eb98-4016-aebe-ae8a99fb13a9]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:37.912581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d1cb6ea-b8c8-4ec3-b35e-07688765aecd]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:37.913317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:37.913665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:37.913995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:37.914343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:50:37.929674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:37.929970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bbd044a-562c-4c28-a764-dd22ad97eca4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:37.933232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d667f063-43f5-47dd-988a-a422bdc0bb93]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:37.934874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:37.935518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:50:37.935870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:37.936524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:37.936911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:37.937323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:37.937701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:37.995956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b25ed21b-0947-4b3b-992b-5a3209c1e3d5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:38.031656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[72b9dfa0-f707-47a1-9387-8a144035ee8e]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:38.031656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:38.031656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:38.031656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:38.031656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:38.031656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:38.031656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:38.031656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:38.031656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:38.031656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:38.031656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:38.031656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:38.031656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:38.032589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72b9dfa0-f707-47a1-9387-8a144035ee8e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:38.033334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:38.034825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:38.037285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3dc9401b-1c78-4571-aef8-b3bd2e7d2fed]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:38.037285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:38.037285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:38.037285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:38.037285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:38.038639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37617d60-5fd4-456d-9f8d-5d86db00cf89]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:38.039706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcdefc70-c35f-4a50-962a-bb3f61f3fc1b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:38.040871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5e1b029-0be9-48c9-8613-165add4730be]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:38.062794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a24b6d0b-0467-4562-a515-43f3df4ed7ca]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 753407, 'reachable_time': 17576, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219273, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:38.070018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:38.070284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba29b5e6-e56e-469d-8bf6-5a90865bd0e4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:38.071333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:36Z", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:38.073938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id d2f2a74d-523c-4089-a17f-c126d536b88e. Wait time elapsed: 5.546 Jul 01 10:50:38.088361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id d2f2a74d-523c-4089-a17f-c126d536b88e. Wait time elapsed: 5.546 Jul 01 10:50:38.088361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:40.167956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.095s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:40.169161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:40.170058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:40.170552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:40.170903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:40.172244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e282b21c-7d91-4e85-ba82-d0749d294ec0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.188305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c17c0255-cf3d-470b-a3a5-4480bc69726e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.212413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e23fc063-58c1-48cb-8617-9807d8a731e3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.242281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c81e7c13-1de4-4d35-aed4-7cde87a6799c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219278, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.263718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93d47f51-d592-41d1-9853-169dd23c2bcf]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.284017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b12a0017-b612-4485-881d-33f392749c17]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.311524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbc07ee0-06f6-44e1-81fa-9f8fc538c8e0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.333207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa1588e1-2c24-4ec1-8f4c-36591a0b6771]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.333822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:40.334303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:40.334771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:40.335325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:50:40.352594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:40.352940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcb9f4f6-da91-459a-ab75-dbba78a12c50]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.358464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a37a962-12f3-43aa-99f0-998f57fda4b4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.359552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:40.412184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:40.412991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:40.413392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:40.413785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:40.414173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:40.478719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4afaff8f-821b-47aa-aff7-230ddc05efa4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.518199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bef714b9-aff7-4750-b9d7-15ab9d86b985]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:40.518199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:40.518199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:40.518199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:40.518199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:40.518199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:40.518199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:40.518199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:40.518199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:40.518199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:40.518199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:40.518199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:40.518199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:40.520053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bef714b9-aff7-4750-b9d7-15ab9d86b985]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:40.521171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:40.524043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:40.526519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78def2e3-025d-46c9-aef7-960021bcd2f3]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.526519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:40.526918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:40.526918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:40.527123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:40.528465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[591ca264-7724-4eea-aea6-a5241ca2b1a1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.530418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94e972cb-c7e8-4cd1-8312-8041597200f9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.531511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a29a94c-923b-4272-be56-38aec9712374]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.555071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4065c50-7f40-447a-9079-868d0ed145ab]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 753650, 'reachable_time': 38624, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219298, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.557152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:40.557447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d816ea66-1644-4d13-8083-6c367efb17c6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:40.558437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:36Z", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:40.561842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id fb997476-0c94-49c8-acdc-c1c770fa63fa. Wait time elapsed: 5.046 Jul 01 10:50:40.573675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id fb997476-0c94-49c8-acdc-c1c770fa63fa. Wait time elapsed: 5.046 Jul 01 10:50:40.573675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:41.261113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.700s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:41.261761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:41.262407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:41.262727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:41.263380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:41.268810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[955d5aaa-3a33-46cf-9b45-51e00c5a1f56]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.289214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f691d7c-33d3-434f-a4fe-24340ddf7984]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.311340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e18afd92-567f-4497-b46d-d006dfcb4b60]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.347838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6627b019-0c75-4051-a914-8fd06881c811]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219301, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.369023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9fbf81be-d0bf-49d9-86f8-7bb6a569b158]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.389768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9efb45cc-8fcc-4d76-8d46-3a2e8e10db32]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.410686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69766d66-1f65-48ce-b910-9ca8faed0f3b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.431613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[016b322e-81f3-4632-a9b0-862dd96dad7d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.432362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:41.432640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:41.433080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:41.433296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:50:41.449030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:41.449427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93617af9-fd7c-44e2-a306-eb5629105316]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.453944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3d74954-68e0-4454-bbc1-be966b3484c0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.454975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:41.489731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:41.490861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:41.491338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:41.491836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:41.492353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:41.558218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae3d82d5-e34d-4143-b070-7b8ee4837e57]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.588474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b27bb20e-b69c-4b95-a0d3-c4bf9f39245d]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:41.588474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:41.588474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:41.588474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:41.588474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:41.588474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:41.588474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:41.588474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:41.588474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:41.588474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:41.588474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:41.588474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:41.588474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:41.589526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b27bb20e-b69c-4b95-a0d3-c4bf9f39245d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:41.589673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:41.591563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:41.594829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96fab2d2-44aa-4656-add6-e40525aaaa45]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.595758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:41.596092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:41.596344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:41.596702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:41.597734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34df41a8-bfb6-4248-8df5-2c258e64fba4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.598702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47fb9328-d642-4a31-9bca-5bff1d82e92a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.599741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[189ee809-6ac9-4dd9-8b89-f0f12c9c441b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.620378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5864ded-bf7e-472e-8aaa-3cd9a9ac41ca]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 753761, 'reachable_time': 30908, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219319, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6768ce4b-d884-4eab-a3ec-380f12af2f8b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:41.627038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:39Z", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id d2f2a74d-523c-4089-a17f-c126d536b88e. Wait time elapsed: 9.100 Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id d2f2a74d-523c-4089-a17f-c126d536b88e. Wait time elapsed: 9.100 Jul 01 10:50:41.631907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:44.320139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.693s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:44.320760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:44.321344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:44.321663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:44.321942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:44.323768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d303536-65e7-4bec-b2d3-5771f953dd1f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.336523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e70cf5f-99e9-46a6-9940-3bf14c771cf9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.363560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc89e04f-633b-48fc-a869-02eae823d87b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.394533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2a66caa-7a11-4f51-84a8-7ff4d43a547e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219369, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.416182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4a5d7b8-5bcd-4ad7-936b-90c113a4ec49]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.437654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28066765-fd15-47ff-8317-ba6e31013ae8]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.457888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5be499ae-87dd-4bdc-a518-fdae2c109b06]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.478869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fff0fb60-3922-4bd1-a893-894b6cb471d0]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.479568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:44.479914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:44.480245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:44.480591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:50:44.496426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:44.496999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63d4c553-7386-4b1e-ae31-6e35c7720df6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.499922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad75157a-eb1b-411f-842a-270622a3f3e7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.500887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:44.532082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:44.532987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:44.533384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:44.533772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:44.534306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:44.583932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab3d5d67-8106-4a95-9fcf-9391ba1202e8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.624366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1a786a4d-c1aa-4b9d-b5c5-fc52a1723662]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:44.624366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:44.624366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:44.624366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:44.624366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:44.624366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:44.624366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:44.624366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:44.624366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:44.624366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:44.624366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:44.624366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:44.624366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:44.625381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a786a4d-c1aa-4b9d-b5c5-fc52a1723662]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:44.626347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:44.628101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:44.630447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f28fd60b-1ca0-40df-9e13-5ed9006c2b26]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.630789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:44.631113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:44.631395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:44.631751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:44.632778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee9f7eaa-f81d-40c0-8509-c030bb28a6ed]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.633699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c22b0dbc-4868-48f2-8089-240118c61b04]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.634605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bf0eef0-2144-47b6-8775-453987672343]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.653977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bef4be4-2f7d-4000-9c49-747ff22f376d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 754065, 'reachable_time': 34076, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219387, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.656658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:44.657068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b94242eb-46c6-4997-8a5b-207f173c0d7d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:44.659077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:39Z", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:44.661928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id fb997476-0c94-49c8-acdc-c1c770fa63fa. Wait time elapsed: 9.146 Jul 01 10:50:44.676240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id fb997476-0c94-49c8-acdc-c1c770fa63fa. Wait time elapsed: 9.146 Jul 01 10:50:44.676240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:45.453326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.792s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:45.453894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:45.454655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:45.454982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:45.455270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:45.456536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[126e4382-2279-43b1-b6c2-6e10012d13fe]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.470856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9022e36-763e-4f9f-a995-29875b1169a2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.491837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecd13223-978e-44ea-9bbf-453b00d02b29]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.522384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90d303c6-54d7-47ce-a3f9-2715848c388c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219390, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.544081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acda8e62-2291-4376-bc3e-5afda2621fa5]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.564850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27fdd146-aab1-4dc2-af27-e2b51f0e3648]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.584774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e74cf55-bc41-48e3-ac6d-48323471412f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.605326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7b6bdc4-ed66-46f4-b0bd-2eb233e96ff9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.606437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:45.607064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:45.607334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:45.607638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:50:45.623752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:45.624054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa7cf93d-7582-4ce7-b617-af590bd688c1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.626721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b859fba5-5b31-4d0b-a1be-6f61c4ab0bbd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.627741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:45.664804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:45.665804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:45.666425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:45.667070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:45.668158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:45.737778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed08cc29-3116-4151-9625-def7f13b6ff1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.774503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3579c2a6-7808-472e-93ff-50e46cf14149]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:45.774503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:45.774503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:45.774503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:45.774503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:45.774503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:45.774503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:45.774503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:45.774503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:45.774503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:45.774503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:45.774503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:45.774503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:45.775623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3579c2a6-7808-472e-93ff-50e46cf14149]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:45.775738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:45.777247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:45.779726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5a6de94-fd5e-432e-b4fb-f490013bd90f]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.779726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:45.779726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:45.780059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:45.780264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:45.782125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de096f9f-1be4-404c-b1ea-2c3fd607fb4d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.783068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa54a5b9-46e2-4ad1-a85f-cbc20cd50dc6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.784001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5e9f785-03ee-432c-b33c-9162c89b39a8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.807528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03f2e798-dfb7-4ba1-b1fd-ca428ba2d736]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 754178, 'reachable_time': 19620, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219409, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.814734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:45.815091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2222631-2813-4a5e-8c42-8166af922c5f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:45.816215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:44Z", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:45.818795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id d2f2a74d-523c-4089-a17f-c126d536b88e. Wait time elapsed: 13.291 Jul 01 10:50:45.832572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id d2f2a74d-523c-4089-a17f-c126d536b88e. Wait time elapsed: 13.292 Jul 01 10:50:45.832572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:49.046995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.229s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:49.047608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:49.048209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:49.048536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:49.048821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:49.050055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7e8fac4-f337-4793-a648-1e13547541e1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.068590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84f298fe-5d81-4126-829d-6fe23cf907d5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.089371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09d26f39-f969-45bd-a634-2e4516ca7c2c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.116999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26f43494-b42a-4d94-841d-7832776cf64c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219462, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.140599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c769031-7fe4-4f29-adb9-2f11eeb0c0fc]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.162223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c48261e-0acb-4d78-8124-4822c87c3d89]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.185019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7976234-6259-49fd-865f-46b533dbc57d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.206900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[759e1c31-8bcf-404c-ac76-c9642c427207]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.207563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:49.207848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:49.208124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:49.208406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:50:49.224360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:49.224895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[06ada2d8-64b5-486c-a100-e355a94787f5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.228284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aefaded4-0662-431e-a024-c73dc5393db2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.229331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:49.262899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:49.263926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:49.264425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:49.264887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:49.265341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:49.323509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[201ff09e-5740-45bd-a9e3-5e1a1abd1d3f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.361419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e33c1db3-491a-439e-9e98-55513bee2c35]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:49.361419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:49.361419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:49.361419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:49.361419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:49.361419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:49.361419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:49.361419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:49.361419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:49.361419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:49.361419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:49.361419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:49.361419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:49.362458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e33c1db3-491a-439e-9e98-55513bee2c35]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:49.363318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:49.364813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:49.368828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[467a900f-3b3e-411b-a725-a2d7a6e83633]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.369916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:49.370262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:49.370668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:49.371164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:49.372830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ded11440-5e50-4ae1-824c-02c5303caedc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.373801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5abf7c1e-a452-4f85-a6cf-c614eb758730]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.374749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8687eebf-4ba5-4c03-af49-515e90d4e4f8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.396609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d04c8a6-967f-49b3-b4c0-2b603ff95464]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 754538, 'reachable_time': 33464, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219480, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.405295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:49.405795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bba42e94-e954-4933-a01c-646c7ab5c41f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:49.406934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:47Z", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:49.409506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id fb997476-0c94-49c8-acdc-c1c770fa63fa. Wait time elapsed: 13.894 Jul 01 10:50:49.425578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id fb997476-0c94-49c8-acdc-c1c770fa63fa. Wait time elapsed: 13.894 Jul 01 10:50:49.425578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:50.132273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.723s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:50.132814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:50.133626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:50.134049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:50.134545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:50.136320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ab2329a-ab92-4471-afba-c77940c137ed]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.155969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fc6b719-e9f0-4460-93d2-cc4905fc4ae0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.196285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23acc090-ebe5-4bd0-92ed-37b9566d7a93]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04f8675d-3daa-4a9b-9488-042893f85159]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219484, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.248670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[666eae75-1482-4f26-b569-cbd7cc0c65a6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.278234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[451877bf-fd02-4796-a088-7fa9dcca1dea]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.298844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78761af4-92ac-49a7-9468-ec1b59abc5bb]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.319016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f0d31c8-a831-4870-8675-5c62f4073ad8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.319616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:50.319891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:50.320157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:50.320431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:50:50.336405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:50.336750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a6834db-c427-43eb-96a8-11e63e933f15]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.339729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d046d74c-384e-485c-b754-97899486481f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.341246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:50.386429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:50.389715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:50.390187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:50.390722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:50.391269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:50.451190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c83fe0f6-07a1-4f44-8de5-cce29dabac0c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.490737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bbaa0ce1-b02e-447c-a289-560862bf8e80]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:50.490737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:50.490737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:50.490737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:50.490737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:50.490737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:50.490737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:50.490737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:50.490737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:50.490737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:50.490737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:50.490737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:50.490737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:50.491778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbaa0ce1-b02e-447c-a289-560862bf8e80]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:50.491983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:50.493813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:50.496108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[050e00e7-f4e6-461d-a990-ed3757a69b69]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.497540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:50.498103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:50.498512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:50.498866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:50.499964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5380233f-6c86-430c-8234-728ab6252649]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.501073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb8027c0-c370-417d-93d4-45e6926630ef]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.502816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5d353a8-3617-4419-9a77-043866ace6ca]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.522019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0197ee3-31ee-49f4-a8f9-3fde800b5cd0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 754648, 'reachable_time': 31452, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219502, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[982f4c9a-d47c-4bba-ae66-5a009cf31792]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:50.531219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:49Z", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id d2f2a74d-523c-4089-a17f-c126d536b88e. Wait time elapsed: 18.001 Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id d2f2a74d-523c-4089-a17f-c126d536b88e. Wait time elapsed: 18.001 Jul 01 10:50:50.536445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:52.562503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-2c654cc6-a34e-4cfa-ae48-5725740b0886 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['0fb3819d-deb2-4bcf-ac9d-627e82b9fe54'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:50:53.563882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.036s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:53.564921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:53.565857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:53.566181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:53.566456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:53.567665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8654f7a4-385d-4b11-ac88-79337f02a0b3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.582844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44e12918-8dee-4716-b697-30b979e6464a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.604222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf588609-47b8-4747-a815-da919ba31831]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.631953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7fe8b6d-f734-45f8-b710-59534154796f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219553, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.654177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0975feb7-1ac7-4a06-be16-a740224071a4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.674033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa79169b-5991-43cc-a35a-8cc35d36180a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.694731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c34b0836-5186-44bd-9c6e-9e4141b3f044]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.716093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75e52e2c-d974-4258-8053-19450c8fb6d8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.716607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:53.716881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:53.717164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:53.717439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:50:53.733133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:53.733436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8736be6-f293-4732-a8f5-c5a4c456f882]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.736041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd23c996-bef6-46c3-8fd6-8950c0c1daa8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.737713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:53.775351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:53.776053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:53.776442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:53.777409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:53.778165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:53.829324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51e5d4ce-a707-4288-a9fe-605ca4810775]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.860851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e9bd876c-3128-48ec-b3a1-1e8409d5282a]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:53.860851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:53.860851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:53.860851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:53.860851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:53.860851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:53.860851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:53.860851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:53.860851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:53.860851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:53.860851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:53.860851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:53.860851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:53.862280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9bd876c-3128-48ec-b3a1-1e8409d5282a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:53.863259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:53.864997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:53.867891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e0a18a0-110e-4cee-949a-e88350ecb98a]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.867891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:53.867891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:53.867891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:53.868290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:53.869274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5dcb1bc2-b765-4cef-8480-32aeef3c1727]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.870267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f229e8b-a6be-47e6-8ba2-ec36f80cb2e4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.871747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65f6e688-ca72-4294-903e-c325f8e7f464]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.895647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c9a941c-3fbb-4418-aab2-2984997c2276]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 754989, 'reachable_time': 23812, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219571, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.903747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:53.906617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7f46e47-6387-4814-a976-af2da817e08b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:53.907867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57be813b-3b5b-4418-9a95-982a41b892f3", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.8", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4194a4ef-40df-4f6e-a4d4-58bb7c1dd784", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6484, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:da:7e:0d", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "ecb2a17d-cf28-4422-9cb5-a157dc5cb5a9", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:51Z", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:51Z", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:53.911933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:52Z", Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 1, update_id e2b2c533-4d15-4f27-8223-3d05babb7673. Wait time elapsed: 1.351 Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 1, update_id e2b2c533-4d15-4f27-8223-3d05babb7673. Wait time elapsed: 1.352 Jul 01 10:50:53.925674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:54.761603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.847s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:54.762912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:54.763606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:54.763935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:54.764219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:54.765471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a8812fe-1de7-4f2e-9e0d-226203aefed9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:54.779960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[698bba0e-e97d-4806-965e-ff3a33dddfd3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:54.801219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c1bab09-aef7-471e-85bf-20ab4b4d2acb]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:54.833467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b1b8bb2-0483-43c5-ac3d-bbffc2c20fcb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219575, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:54.858375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d45dbec-19ff-4c64-a100-e10e6d83ee7c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:54.879745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2280280-6ff6-4ae8-978a-d5cae85e9fc9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:54.901116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bf8648f-d5ee-47a8-bb22-9c5435db8863]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:54.927625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38d49735-5e67-4ed1-afcb-f107f5a0ae57]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:54.928271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:54.928609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:54.928881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:54.929155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:50:54.946183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:54.946477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2592aa5a-7ba2-4d99-be2c-f574d482fe85]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:54.949086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3e1fce5-f8a9-42d0-9786-5202899ff428]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:54.950248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:54.995150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:54.995888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:54.997524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:54.997915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:54.998301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:55.062452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7c9ba9c-1499-4e3b-9682-afcf65dc3025]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:55.098662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b9e0dda8-9b7e-4c23-9a47-92cb11cb6c7e]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:55.098662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:55.098662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:55.098662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:55.098662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:55.098662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:55.098662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:55.098662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:55.098662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:55.098662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:55.098662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:55.098662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:55.098662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:55.099659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9e0dda8-9b7e-4c23-9a47-92cb11cb6c7e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:55.100470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:55.103200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:55.105243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21ca647e-d447-4100-9993-5134994ed9f2]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:55.107197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:55.107732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:55.108055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:55.108404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:55.109534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1243a710-f6ce-4123-9711-19510d1f6e46]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:55.111164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f16bf4c-4ccc-43fc-8844-32d4453b1e14]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:55.112367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2abc6fd2-a8aa-4c7e-8396-f84c2aed7691]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:55.134001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c9c2f45-ab7b-4c3a-ac77-5aeb31783f6c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 755109, 'reachable_time': 27332, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219594, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:55.136207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:55.136710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f90f6b3d-13dd-4d2e-b202-39c92c7bb989]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:55.138961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57be813b-3b5b-4418-9a95-982a41b892f3", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.8", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4194a4ef-40df-4f6e-a4d4-58bb7c1dd784", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6484, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:da:7e:0d", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "ecb2a17d-cf28-4422-9cb5-a157dc5cb5a9", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:51Z", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:51Z", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:55.143902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:53Z", Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id fb997476-0c94-49c8-acdc-c1c770fa63fa. Wait time elapsed: 19.636 Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id fb997476-0c94-49c8-acdc-c1c770fa63fa. Wait time elapsed: 19.637 Jul 01 10:50:55.159812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:55.336041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:50:57.618620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.467s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:57.619250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:57.620314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:57.620643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:57.620924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:57.623477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17fe65e7-c2a4-40e2-934e-3dd9ea7a90f0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:57.640619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[312c0198-2c33-4c8c-be24-e675a769dfda]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:57.661041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd555df8-d00b-4025-8f1a-d3c8269bc1b3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:57.692575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c273162d-8a20-4cfb-817c-1692d9113eb8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219600, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:57.714493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e72f1d5-dc5d-4e5a-b9bb-db9415904ff4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:57.737554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9f8afc8-e3b1-40dc-aaaa-7efa76d525a7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:57.760634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a91e2d6-27fb-4f61-9fcc-54b4497b64a5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:57.783252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d5fa4d5-291f-4994-90a9-cb04b97d5da5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:57.783895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:57.784170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:57.784437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:57.784715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:50:57.804540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:57.805194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d95faa8b-dec5-452c-b567-e589ac697ad7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:57.808206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81d367d7-1090-43d5-aa94-ebbf5d761200]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:57.810493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:57.856875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:57.857985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:57.858656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:57.859285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:57.859881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:57.935072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22d23a85-1442-40bb-bb90-d63acdbbb971]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:58.040974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3bfbc839-23bd-4944-a132-69c12fb23c15]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:58.040974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:58.040974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:58.040974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:58.040974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:58.040974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:58.040974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:58.040974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:58.040974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:58.040974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:58.040974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:58.040974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:58.040974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:58.045761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bfbc839-23bd-4944-a132-69c12fb23c15]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:58.054185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:58.057310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:58.064019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02fb1c98-cd52-47f4-9bae-892af63613e5]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:58.065220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:58.065689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:58.066073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:58.066686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:58.070471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f46c2000-8e74-4cd4-9d5a-92c927c92556]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:58.071458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6fe0b861-7b89-460c-939b-18697d98b90f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:58.074620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a07f0c6-00c2-4aba-ae4c-a2bffc19fe1e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:58.108745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3be11327-77cf-4566-aab6-5c36b0924f6f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 755395, 'reachable_time': 41508, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219652, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:58.120263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:58.120795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3c92df3-d492-4895-80f0-9c663e3eb0df]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:58.126530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57be813b-3b5b-4418-9a95-982a41b892f3", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.8", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4194a4ef-40df-4f6e-a4d4-58bb7c1dd784", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6484, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:da:7e:0d", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "ecb2a17d-cf28-4422-9cb5-a157dc5cb5a9", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:51Z", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:51Z", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:58.131323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:54Z", Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id d2f2a74d-523c-4089-a17f-c126d536b88e. Wait time elapsed: 25.606 Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id d2f2a74d-523c-4089-a17f-c126d536b88e. Wait time elapsed: 25.607 Jul 01 10:50:58.150003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:50:59.020538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.886s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:50:59.021125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:50:59.022076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:50:59.022501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:59.022870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:59.024211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9aa21b87-4e4e-43a2-b604-ce994870d8a0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.043947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ba64fac9-c1b5-4658-80e4-ff4763fa5cee]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.065073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1741e3c-1e78-457e-b313-975a2fc084cb]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.094598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89803eff-f7b9-4678-a0a5-3b09936f9766]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219675, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.116042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9099cea6-0ff9-4905-8982-08620420c2df]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.142646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e174698-9ab5-417d-a1b5-8ffef4a8c870]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.165125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cfd0698a-4eda-40b5-ba5b-8332dc179771]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.186517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91c5b323-2c1d-4114-93fb-ca6c38a402e2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.187186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:50:59.187794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:50:59.188258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:50:59.188598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:50:59.207591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:50:59.208085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55441249-3076-4e9c-b638-c05a1e9a9c9a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.210804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[774e163e-933b-40b1-8dd0-928af5a47063]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.212067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:59.257717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:59.258620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:59.259241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:59.260851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:59.261476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:50:59.343618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ad1f9f0-9854-4b36-8a5b-0be994bf6bcd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.376493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[736a4d74-58b5-4ab3-8ffa-e00e53c80962]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:50:59.376493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:50:59.376493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:50:59.376493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:50:59.376493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:50:59.376493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:50:59.376493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:50:59.376493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:50:59.376493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:50:59.376493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:50:59.376493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:50:59.376493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:50:59.376493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:59.377449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[736a4d74-58b5-4ab3-8ffa-e00e53c80962]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:59.377885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:59.382182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:59.385037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[568050ff-abbd-4e0d-b2b9-d9dba005edef]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.385902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:59.386277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:59.386579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:50:59.387128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:50:59.388298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc024bc6-efe3-497f-9963-70259216af2d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.389487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f48e913c-e1f2-4236-a814-ad0084611735]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.390524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3afa156-1baa-4961-8dda-9bd7623820ee]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.411744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e3e37f4-f6dc-45da-9e1f-564dd47ac9c7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 755535, 'reachable_time': 32380, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219694, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.420176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:50:59.420491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d3a9848-7bd7-4547-b324-a03b9a4a8895]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:50:59.421558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57be813b-3b5b-4418-9a95-982a41b892f3", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.8", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4194a4ef-40df-4f6e-a4d4-58bb7c1dd784", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6484, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:da:7e:0d", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "ecb2a17d-cf28-4422-9cb5-a157dc5cb5a9", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:51Z", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:51Z", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:50:59.425684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:57Z", Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3 Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id e2b2c533-4d15-4f27-8223-3d05babb7673. Wait time elapsed: 6.864 Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id e2b2c533-4d15-4f27-8223-3d05babb7673. Wait time elapsed: 6.864 Jul 01 10:50:59.440948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:51:02.061563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.635s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:51:02.062599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:51:02.063577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:51:02.063922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:02.064220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:02.065498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94661b25-699c-4c06-84b4-d73818c845ac]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.080229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59ea40e7-ca86-468b-aceb-61e221bd2e18]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.100655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e2f5687-bda1-4730-ba3e-88a83f74aefe]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.131547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96a2144c-11a9-477b-b4e9-ceb84476eb34]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219700, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.153144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a786a8b-26c2-45ee-866d-13b9d8fbc691]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.173605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7800dc19-b4a1-4993-bad5-9b719a986540]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.194379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc6bd592-ba9d-42e3-a602-153e0f3a409a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.218741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ded685d3-39e3-4daa-a075-cd8561ec311b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.219444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:02.219786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:02.220115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:51:02.220459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:51:02.236194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:51:02.236501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e273f45-5854-4af0-bcf8-c581f9ccc3f0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.239415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a07bfe48-eeb9-4160-b441-eb7bb8388bf6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.240340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:02.284279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:02.285288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:02.286186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:02.286779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:02.287524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:02.347130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7af4ed2d-2509-446c-b1cd-67b479ae22a5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.387623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4b8c8b7c-9bb5-4b7d-96c8-83224a1a8d3e]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:51:02.387623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:51:02.387623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:51:02.387623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:51:02.387623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:51:02.387623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:51:02.387623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:51:02.387623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:51:02.387623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:51:02.387623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:51:02.387623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:51:02.387623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:51:02.387623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:02.388709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b8c8b7c-9bb5-4b7d-96c8-83224a1a8d3e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:02.388860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:02.390492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:51:02.392574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[519144dd-511e-48ec-abe3-325a306ef012]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.392574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:02.392574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:02.392574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:51:02.392952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:02.394827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd98a82b-47dc-4dc5-ab6b-130deaaa7668]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.395942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[204f4553-e839-41ce-95f1-225e9c5956c2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.397214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[327586bc-463e-4027-8637-008d736376a2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.420680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f60ded72-47e8-4967-a63f-538d41635c68]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 755839, 'reachable_time': 41304, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219718, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.431900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:51:02.432440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16ef7b78-756d-44f3-afdc-1856ac4ef3bc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:02.435060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57be813b-3b5b-4418-9a95-982a41b892f3", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.8", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4194a4ef-40df-4f6e-a4d4-58bb7c1dd784", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6484, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:da:7e:0d", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "ecb2a17d-cf28-4422-9cb5-a157dc5cb5a9", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:51Z", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:51Z", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:51:02.437886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:51:00Z", Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id fb997476-0c94-49c8-acdc-c1c770fa63fa. Wait time elapsed: 26.922 Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id fb997476-0c94-49c8-acdc-c1c770fa63fa. Wait time elapsed: 26.922 Jul 01 10:51:02.454496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:51:03.501867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.065s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:51:03.502482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:51:03.503029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:51:03.503345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:03.503672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:03.504825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ebdc92a7-efd6-46d9-80d8-eb80d17a9bb9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.516911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb25350c-d03f-4dfc-9aa0-f8580312cfbd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.537715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[141b76ca-9b77-4f32-99fa-70b9e6623456]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.565028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[062741d7-1a71-4130-ade4-d5ba21600349]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219767, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.585543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f464ea2-fce5-4769-9edd-ac476161fded]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.607169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c6be871-9db8-4e71-8dcc-865d4cee5b82]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.627746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3e9ae99-d6d7-4816-81bc-8a9b3f4e71ee]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.655246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ba1d2c7-66e7-426f-bb83-ae3062ce9d06]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.656107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:03.656358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:03.656646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:51:03.656952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:51:03.683083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:51:03.683664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a322aab-0e13-40b6-9669-969003c20be0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.688096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c50a1d5-4187-4771-9197-6261656140e5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.689102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:03.731976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:03.732658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:03.733226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:03.733997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:03.734390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:03.796937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0385a246-82b8-4b9d-9432-f5654baf25e9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.834980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7b44a65f-fa5f-405d-bf9d-706974bbb8a0]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:51:03.834980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:51:03.834980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:51:03.834980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:51:03.834980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:51:03.834980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:51:03.834980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:51:03.834980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:51:03.834980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:51:03.834980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:51:03.834980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:51:03.834980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:51:03.834980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:03.836338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b44a65f-fa5f-405d-bf9d-706974bbb8a0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:03.837251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:03.838858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:51:03.841646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f821ef7a-f042-4db5-9929-afe897a8bb58]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.841646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:03.841646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:03.841646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:51:03.841965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:03.842781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f401f4a2-1944-4589-aa38-7a4e116ae059]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.843788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1953a07-a331-4196-952c-968e1f42dd23]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.844896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ec5674f-8a1b-48b5-9688-69ce435bb6ba]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.865910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[882aee9d-ec2c-4b54-9af8-93363370a869]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 755982, 'reachable_time': 38948, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219786, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.868058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:51:03.868527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4786d70f-cd54-4496-9bf7-fdb6751014ce]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:03.869699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57be813b-3b5b-4418-9a95-982a41b892f3", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.8", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4194a4ef-40df-4f6e-a4d4-58bb7c1dd784", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6484, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:da:7e:0d", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "ecb2a17d-cf28-4422-9cb5-a157dc5cb5a9", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:51Z", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:51Z", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:51:03.875352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:51:02Z", Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 21, Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3 Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id e2b2c533-4d15-4f27-8223-3d05babb7673. Wait time elapsed: 11.318 Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id e2b2c533-4d15-4f27-8223-3d05babb7673. Wait time elapsed: 11.319 Jul 01 10:51:03.889612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:51:06.465707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.583s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:51:06.466383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:51:06.467114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:51:06.467463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:06.467771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:06.468925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2802e43-20f7-41ed-be93-0cba94ad0a93]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.484198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3dc451c2-7b95-40de-a10c-659c24990dfa]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.506058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac71b9b1-ba98-4c66-99ff-793c6cbb49f1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.534330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e79e6261-ba95-4178-b814-58edd01d9962]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219791, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.558669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b9a44c1-7ec3-4c32-b421-8a9680de5a16]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.585684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c31d12b-af03-499d-a6db-8af811de31d7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.606876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30d16286-26b5-47b1-93d5-9dc4c47ee58c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.627292 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aaacf2b9-5d97-4f94-86f9-8b4048d95678]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.628011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:06.628290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:06.628568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:51:06.628826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:51:06.644500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:51:06.644808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28c4f438-b664-4664-8ea8-2294522032f8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.647557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36f7d956-a983-4a25-b004-bba12a75f30d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.648619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:06.688104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:06.688898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:06.689365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:06.689827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:06.690291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:06.742458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5dd04aa7-6bf0-402e-bb60-93a3f91e5a77]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.783512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a536e5bf-b687-471c-b3cd-dd1d6218d4e3]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:51:06.783512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:51:06.783512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:51:06.783512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:51:06.783512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:51:06.783512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:51:06.783512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:51:06.783512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:51:06.783512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:51:06.783512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:51:06.783512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:51:06.783512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:51:06.783512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:06.784632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a536e5bf-b687-471c-b3cd-dd1d6218d4e3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:06.785326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:06.787115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:51:06.789560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5b320928-0e7f-4904-9e6b-e1707edb5e3e]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.789560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:06.789560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:06.789560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:51:06.789926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:06.790816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[943f5639-489e-4320-9ab0-332f2455b8b2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.791843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67fd94f8-e2fa-4da2-a77e-4e096af2b131]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.792972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[036a037e-af0c-4792-acc1-e48b7c2fbf8d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.813342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[663e7e9b-c0bf-45ae-b3eb-3f0a84ff5e4f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 756279, 'reachable_time': 43180, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219810, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f819f15-b1d4-4f44-a2a8-a6a37b5acb85]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:51:06.817738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57be813b-3b5b-4418-9a95-982a41b892f3", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.8", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4194a4ef-40df-4f6e-a4d4-58bb7c1dd784", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6484, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:da:7e:0d", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "ecb2a17d-cf28-4422-9cb5-a157dc5cb5a9", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:51Z", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:51Z", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:51:04Z", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:51:06.822508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id e2b2c533-4d15-4f27-8223-3d05babb7673. Wait time elapsed: 14.255 Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id e2b2c533-4d15-4f27-8223-3d05babb7673. Wait time elapsed: 14.255 Jul 01 10:51:06.834706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:51:07.568157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.749s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:51:07.568749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:51:07.570369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:51:07.570773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:07.571063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:07.572601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a8fabf4-4045-4d33-af8d-c4e6a2ac670f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.586528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a33c56a-4095-46de-abe3-0516e01650ab]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.607780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85da1210-bea5-4444-b0ce-a1a73accc8ef]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.637154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4bbdb77a-1176-4a3e-b273-aac45299710f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219813, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.659080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a761ad36-a139-49ce-8cca-b6c311f2997c]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.683333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea9ffba8-63eb-46d7-998c-d6b4d4cd7f33]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.706483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63e05e24-8cfb-4ca7-a6b2-49e7e2528a77]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.727824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b127dd73-392d-47d3-8fb3-c540d08710f2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.728397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:07.728678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:07.728942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:51:07.729216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:51:07.744992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:51:07.745300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04bcf8d3-396b-48e5-8455-029cdcab53b4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.748951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d04084dd-7bce-4e3f-bb75-c520a7fc8a8f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.749887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:07.791624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:07.792681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:07.794040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:07.794540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:07.795031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:07.858253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68491707-4e79-4e1f-8666-e00ef4c29264]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.913208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[13d51c57-b70d-4a3e-9741-0506405253a3]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:51:07.913208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:51:07.913208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:51:07.913208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:51:07.913208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:51:07.913208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:51:07.913208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:51:07.913208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:51:07.913208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:51:07.913208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:51:07.913208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:51:07.913208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:51:07.913208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:07.914149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13d51c57-b70d-4a3e-9741-0506405253a3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:07.914453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:07.917145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:51:07.920198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42dd690a-fd50-4e69-80d3-9b994e747018]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.920198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:07.920416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:07.920683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:51:07.921284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:07.922954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2572a8a9-a945-4407-8e8f-cd5270565ec2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.924137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eeb72d02-180e-443d-a2b9-f7df467a841f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.925540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc9ff9e5-e606-47c8-946a-4b63e7e3b7b4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.958608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bc63581-0ea9-445c-8062-16f3f9a42af4]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 756390, 'reachable_time': 43736, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219836, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.970309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:51:07.970711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf77cc49-9c48-4cc1-9578-057ca4af4c78]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:07.971838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57be813b-3b5b-4418-9a95-982a41b892f3", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.8", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4194a4ef-40df-4f6e-a4d4-58bb7c1dd784", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6484, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:da:7e:0d", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "ecb2a17d-cf28-4422-9cb5-a157dc5cb5a9", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:51Z", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:51Z", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:51:07.974570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:51:06Z", Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 25, Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id e2b2c533-4d15-4f27-8223-3d05babb7673. Wait time elapsed: 15.410 Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id e2b2c533-4d15-4f27-8223-3d05babb7673. Wait time elapsed: 15.410 Jul 01 10:51:07.992353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:51:10.418166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.444s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:51:10.418814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:51:10.420544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:51:10.420878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:10.421168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:10.422285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d767b76b-57b1-415f-87cd-06c2a4edae91]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.440830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4711496d-014c-4621-8341-c184c09738a3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.462139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca53e409-19f6-4525-95bd-89bac863dfa1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.489833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc665ea7-6782-46e0-9f04-918a5a051922]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219886, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.511891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82c0325c-01f8-4b05-8feb-9cfac2d837b0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.532118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec0ebde2-9555-415a-bd6f-f09ef3232e08]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.552305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c65031a-4348-44b3-a118-50556ce58427]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.580357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bed839ca-7322-42b7-8033-80c45ba88001]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.580916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:10.581186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:10.581448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:51:10.581720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:51:10.600061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:51:10.600355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0668f399-25d1-42a3-b8cd-679d3fd01308]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.602925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ec7aa76-1b28-445f-a62c-f2c2190b350c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.604478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:10.647301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:10.648536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:10.649093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:10.649768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:10.650735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:10.727647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6a70565-f4fd-4b78-aeaf-0c3568850801]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.762498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[459dd7b8-09dd-402a-bfe5-fec4604bcdd5]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:51:10.762498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:51:10.762498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:51:10.762498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:51:10.762498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:51:10.762498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:51:10.762498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:51:10.762498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:51:10.762498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:51:10.762498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:51:10.762498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:51:10.762498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:51:10.762498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:10.764293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[459dd7b8-09dd-402a-bfe5-fec4604bcdd5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:10.765283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:10.766836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:51:10.769196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e18d7d9-97ca-410f-ada9-339eb66d39e6]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.769796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:10.770118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:10.770869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:51:10.771211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:10.772284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[042794bb-9582-4718-b57d-06ff30acc468]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.773298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ae5a31f-5cb6-4c37-8912-1a8e4cf6ecf9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.774208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[206e0945-9fd1-483c-9af7-b8147a430bfa]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.793654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5a49f0b-2697-40c7-a33c-86f3fe384a42]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 756675, 'reachable_time': 36028, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219904, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.795895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:51:10.796319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b77c71e-d5fd-4f51-b717-21cc43794e5b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:10.797680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57be813b-3b5b-4418-9a95-982a41b892f3", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.8", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4194a4ef-40df-4f6e-a4d4-58bb7c1dd784", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6484, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:da:7e:0d", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "ecb2a17d-cf28-4422-9cb5-a157dc5cb5a9", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:51Z", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:51Z", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:51:10.802598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:51:09Z", Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 27, Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id e2b2c533-4d15-4f27-8223-3d05babb7673. Wait time elapsed: 18.240 Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id e2b2c533-4d15-4f27-8223-3d05babb7673. Wait time elapsed: 18.241 Jul 01 10:51:10.822890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:51:11.631185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.827s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:51:11.631833 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:51:11.632542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:51:11.632875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:11.633161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:11.634449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6f28352-5eb8-4e27-92b6-52cff19d696e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.651915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49e3abe6-af35-4734-bdfe-29a1198f5f29]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.673721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd828c6f-0e9a-4e58-8398-2b9ab078df4b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.702681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[326a5fd2-963f-4693-a7a5-19cd645104b6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 219907, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.723536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a003c0e4-e093-44af-b84a-168cfcf2971a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.746437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[944a9799-7710-4a67-9169-3daba912bc50]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.770624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[279b7dbc-bf04-4d35-b96b-6c67f722683c]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.795108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[065340d5-c86f-4694-a2ed-36f77046b718]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.795723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:11.796008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:11.797837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:51:11.797837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:51:11.812923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:51:11.813312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08905f00-b592-46aa-beca-f1b6abb12261]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.816173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36214e17-e8e7-4862-b189-2adf06d6bd29]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.817181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:11.861887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:11.863842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:11.864497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:11.865439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:11.866080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:51:11.920338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3060836-9b84-4290-b8bf-37cdbec47cb7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.951180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b74f4bce-2b79-48d5-91e3-6e0df0c22d0e]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:51:11.951180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:51:11.951180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:51:11.951180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:51:11.951180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:51:11.951180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:51:11.951180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:51:11.951180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:51:11.951180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:51:11.951180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:51:11.951180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:51:11.951180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:51:11.951180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:11.953642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b74f4bce-2b79-48d5-91e3-6e0df0c22d0e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:11.957649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:11.959629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:51:11.961918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4429f55b-8311-41bf-8eb1-72a788e1561b]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.961918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:11.962144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:11.962421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:51:11.962698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:51:11.963810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2ee00c1-0eee-491e-893d-19994d74ff0e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.965413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc0ffa7c-6138-4b4e-8b36-94d37c8c1a74]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.966398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a32d379-90d6-457c-bdc2-f07a478715ee]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.987698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94a64302-c975-44a1-868d-3d78d77df9a0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 756796, 'reachable_time': 15844, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 219926, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.989991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:51:11.990492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e7db7b4-cb0f-4a69-b1fa-f599a0f02785]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:51:11.991784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "57be813b-3b5b-4418-9a95-982a41b892f3", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.8", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "4194a4ef-40df-4f6e-a4d4-58bb7c1dd784", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.11", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6484, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:da:7e:0d", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "ecb2a17d-cf28-4422-9cb5-a157dc5cb5a9", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:51Z", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:51Z", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:51:11.996582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:51:10Z", Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:51:12.008180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3 Jul 01 10:51:15.162801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:15.163579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:15.163772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:51:15.164009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:51:35.342711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:52:15.163517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:52:15.164196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:52:15.164196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:52:15.164386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:52:15.347109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:52:55.348478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:53:15.164704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:53:15.165402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:53:15.165402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:53:15.165603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:53:35.349694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:54:15.165699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:54:15.166403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:54:15.166403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:54:15.166598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:54:15.350803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:54:55.352402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:55:15.165102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:55:15.165790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:55:15.165790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:55:15.166011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:55:35.353810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:55:54.381166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c9cd718b-527e-417d-9ddd-b1cb93247b7d tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['0fb3819d-deb2-4bcf-ac9d-627e82b9fe54'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:55:54.382916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 1, update_id 2dd31447-400f-4b51-9c28-f3480c689a16. Wait time elapsed: 0.001 Jul 01 10:55:54.383185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 1, update_id 2dd31447-400f-4b51-9c28-f3480c689a16. Wait time elapsed: 0.002 Jul 01 10:55:54.384135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:55:56.494426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.111s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:55:56.495062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:55:56.495795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:55:56.496294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:55:56.496579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:55:56.497750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6076e864-37b9-4b15-8a03-b69d1b69f809]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.511711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e692fd3-0db8-43ed-89d5-99160f0a682a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.533156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1e9eb72-8de2-493a-b651-381bd1d46633]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.579965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de929ac9-9f18-40ac-b168-896fecb457a8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 224780, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.600869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc35e509-4c55-4b1e-ac2b-5511cb12b75e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.629846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6093a296-4d1c-401d-8600-4f8604d484b5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.651204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cbd46398-2987-46c1-8927-668d98d1f5aa]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.671222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b843c363-6976-4cf2-8680-73ec26542d84]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.671818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:55:56.672097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:55:56.672363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:55:56.672645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:55:56.689030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:55:56.689235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2eb2fda0-51f8-47ef-8e95-0ea73a8d987b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.692020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4284cac-32e9-47c5-b82b-92629f9f9a32]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.693040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:55:56.736469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:55:56.737639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:55:56.738315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:55:56.738883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:55:56.739457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:55:56.778499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6438c339-d63f-4254-a08f-acb2ed17bffb]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.815978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d3119548-d517-4efb-9117-ab0325054107]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:55:56.815978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:55:56.815978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:55:56.815978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:55:56.815978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:55:56.815978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:55:56.815978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:55:56.815978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:55:56.815978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:55:56.815978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:55:56.815978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:55:56.815978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:55:56.815978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3119548-d517-4efb-9117-ab0325054107]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:56.817081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:56.820003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:55:56.822981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[515f9a85-702e-463d-8023-5afd39b48a9f]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.822981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:55:56.822981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:55:56.822981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:55:56.822981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:55:56.822981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[967dee3f-e176-4ac7-ad2b-f783a6288bdb]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.822981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[820f26cd-b646-4417-93a9-2acde0dc992a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.823876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11a2f095-2e69-4ed5-bad3-cfc81944dd0b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.843188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d112288-cea9-4def-89b9-a61ca04af050]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 785284, 'reachable_time': 39476, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 224799, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.849363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:55:56.849718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0ef2a55-57ef-4982-97bb-389c8ef026ae]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:56.851459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:51:14Z", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 33, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:55:56.857423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id 2dd31447-400f-4b51-9c28-f3480c689a16. Wait time elapsed: 2.472 Jul 01 10:55:56.873962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id 2dd31447-400f-4b51-9c28-f3480c689a16. Wait time elapsed: 2.472 Jul 01 10:55:56.873962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:55:57.873111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.018s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:55:57.873769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:55:57.874418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:55:57.875075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:55:57.875450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:55:57.876952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c16be40-dc10-4770-b4fe-80f6d8dbaf89]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:57.898291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8fd9a242-337a-4052-88b1-ae26b4c63b1f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:57.923600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[775ca7d3-7bd5-4640-a3c4-8e66d46e962a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:57.961207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[205dd1e8-ac37-4b5f-9e30-6c7fd360c24b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 224850, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:57.985963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47dc284e-3308-4ed6-b633-66f806a0923e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.019588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9407f14-041d-4cb1-bf02-bf3b981b58e9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.056663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[154d457e-8407-49ef-bd2d-42e9ae1035c0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.087776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d597312-b6eb-4e22-97a7-c1c619a30afa]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.088813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:55:58.089438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:55:58.089996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:55:58.090706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(37ef0172-6291-4d78-92ec-635ac5c8827d) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:55:58.121050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:55:58.121560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fc1224e-9a32-46f2-86d3-0ea71024db45]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.125781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[adf288a0-2868-4b58-8bc4-ecd6cb0deadb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.127415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-37ef0172-62, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:55:58.174755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-37ef0172-62, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:55:58.191597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:55:58.192362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:55:58.192966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-37ef0172-62, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:55:58.193568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-37ef0172-62, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '37ef0172-6291-4d78-92ec-635ac5c8827d', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:95:ff:e5'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:55:58.265293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[586c4a10-349c-49a8-a313-648aeb5e1dd0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.307327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[12f97fac-cd2f-4baa-bfc1-e26e22dc5588]: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:55:58.307327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:55:58.307327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:55:58.307327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:55:58.307327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:55:58.307327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:55:58.307327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:55:58.307327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:55:58.307327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:55:58.307327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:55:58.307327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:55:58.307327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:55:58.307327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:58.308447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12f97fac-cd2f-4baa-bfc1-e26e22dc5588]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:58.308973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:58.310710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:55:58.313795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ed3635a-1513-478b-ac64-bcbaf6d30f0d]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.313795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:55:58.313795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:55:58.313795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:55:58.313795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:55:58.315265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77bc013a-a31a-4905-bdac-b8f6876437e4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.316305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5aed867e-b329-4be4-b7f2-bced735b95f2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.317264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[665ea114-630c-47c7-9a02-7a924b9621b3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.340003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc386263-14cc-42c8-aaea-2b9e9af42160]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 785422, 'reachable_time': 19660, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 224870, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.347884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:55:58.348175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0db49994-53d7-4252-a9bc-ad8e297d4d5e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-37ef0172-62 not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:55:58.349181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:34Z", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "37ef0172-6291-4d78-92ec-635ac5c8827d", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:95:ff:e5", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6469, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:34Z", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:51:14Z", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 33, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "dd36237e-c86f-4931-9cce-6bf6f6d07e52", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:55:58.353206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id 2dd31447-400f-4b51-9c28-f3480c689a16. Wait time elapsed: 3.969 Jul 01 10:55:58.374363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 2, update_id 2dd31447-400f-4b51-9c28-f3480c689a16. Wait time elapsed: 3.970 Jul 01 10:55:58.374363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:55:59.140528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-3adfc6b7-a2a5-43be-b725-c4880962039d tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['0fb3819d-deb2-4bcf-ac9d-627e82b9fe54'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:56:00.175818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.824s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:00.175818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:00.177433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (0fb3819d-deb2-4bcf-ac9d-627e82b9fe54), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:00.178165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:00.178728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:00.181945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9846b98-3ebd-4e0c-99ee-3b1f75a88437]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.212160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-81049c9d-86e2-4acf-a85c-3fea011a616b tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got router deleted notification for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 10:56:00.213297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ce4a5e78-a353-46b0-9dbc-d92f84eff96d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.240339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[daf647d5-2797-4539-8410-fce2547f0e3b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.275914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0753347-4e71-414e-a449-29d959bd95b4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 224880, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.298483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ca6098c-150f-487c-8dbd-53d381745061]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.323596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a2da474-203d-43b8-b1cb-f612a2399400]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.347705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40ef1169-8bc1-477b-8592-800f2c2e42c4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.373594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65b2d179-5856-487d-87cb-cf2d4c7357ed]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.374655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:00.375818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:00.376709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:00.398541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2df8182b-beb5-442c-96e1-24031f79352e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 785654, 'reachable_time': 36436, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 224886, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.400154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:00.400418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:00.400908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:56:00.401383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 10:56:00.402017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:00.402325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:00.402581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'b02c4734-0a9b-43bd-9c32-a0ff9aea7f09', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:a5:ec:88', 'admin_state_up': True, 'status': 'ACTIVE', 'device_id': '0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'device_owner': 'network:router_gateway', 'standard_attr_id': 6465, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.20', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:50:31Z', 'updated_at': '2022-07-01T10:50:37Z', 'revision_number': 4, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-b02c4734-0a), ns(qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 10:56:00.427277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:00.428894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f9691d3-f1c8-48d7-8c54-3aa47fe7ddec]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.436268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e349f676-6ba2-48f8-8322-ae5fd1f6e071]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.438082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-b02c4734-0a, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:00.482716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-b02c4734-0a, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:00.484032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-b02c4734-0a, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:00.484669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-b02c4734-0a, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:00.485355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-b02c4734-0a, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:00.485930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-b02c4734-0a, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b02c4734-0a9b-43bd-9c32-a0ff9aea7f09', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a5:ec:88'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:00.538409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5651a2c-9da7-45db-97a7-2b8429e8e501]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.582708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9025876f-fe02-4b9e-9fca-db2fa3c16849]: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:00.582708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:00.582708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:00.582708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:00.582708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:00.582708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:00.582708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:00.582708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:00.582708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:00.582708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:00.582708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:00.582708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:00.582708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:56:00.584236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9025876f-fe02-4b9e-9fca-db2fa3c16849]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.584890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 10:56:00.585411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 10:56:00.612870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93a414c0-5f44-40ec-8509-c765b97cf812]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.786513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72e47243-5bd2-4d69-a09c-f6db36b8031c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.804430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b66aa26-1a78-4c32-9063-37b9da6fffdb]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.998460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6634b153-c9c5-499f-9944-c7ba2ca2a7ca]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:00.998886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 10:56:00.999203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:56:00.999562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 10:56:00.999880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54" released by "process_external" :: held 0.599s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:56:01.000663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:56:01.003735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:01.007199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f964049-0fa1-4e0d-87b6-ddef19997951]: (4, ['qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:01.007199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:01.007199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:01.007199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:01.007199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/0fb3819d-deb2-4bcf-ac9d-627e82b9fe54.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:01.007199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f977e7b0-ae84-4301-9d7a-e82d3d5db25d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:01.008141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[663d525f-3864-4e37-bd37-5c1f43b2f7e1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:01.009249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c258a5cf-ebd8-4c9a-b415-0e87a79ef8e5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:01.031633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd37b037-1865-496b-9450-73d4aa168321]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 785654, 'reachable_time': 36436, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 224903, 'error': None, 'target': 'qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:01.039178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:01.039580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ee090ec-f19c-404d-9f9f-04767ebc1b09]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-b02c4734-0a not found in namespace qrouter-0fb3819d-deb2-4bcf-ac9d-627e82b9fe54. Jul 01 10:56:01.040775 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-1164226476", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20" Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:55:58Z", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b02c4734-0a9b-43bd-9c32-a0ff9aea7f09", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a5:ec:88", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "0fb3819d-deb2-4bcf-ac9d-627e82b9fe54", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6465, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.20", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:50:31Z", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:50:37Z", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 1, update_id ce2210bc-508a-43be-b1d2-67caca19cdc7. Wait time elapsed: 1.901 Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, action 3, priority 1, update_id ce2210bc-508a-43be-b1d2-67caca19cdc7. Wait time elapsed: 1.902 Jul 01 10:56:01.045182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:01.076972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.033s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:01.077269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54 was not found. Performing router cleanup Jul 01 10:56:01.078365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed7cdbb7-929f-4c03-9058-504602d7313e]: (4, ['qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:01.078969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:01.079269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:01.079566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 0fb3819d-deb2-4bcf-ac9d-627e82b9fe54, update_id ce2210bc-508a-43be-b1d2-67caca19cdc7. Time elapsed: 0.037 Jul 01 10:56:06.248323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8ed20f65-9272-4f0e-8118-ce6c034246d8 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['72167c49-ec60-45f9-b96a-8d47b19f2308'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:56:06.250729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 1, update_id 1527f167-1f66-4d7f-9aca-85fe48351129. Wait time elapsed: 0.001 Jul 01 10:56:06.251053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 1, update_id 1527f167-1f66-4d7f-9aca-85fe48351129. Wait time elapsed: 0.002 Jul 01 10:56:06.252255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:08.857541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-d4e19650-b2d9-420d-b45a-d3c980d888cf tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['72167c49-ec60-45f9-b96a-8d47b19f2308'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:56:09.228947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.977s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:09.229458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:09.230413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:09.230903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:09.231658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:09.232983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[626e8839-ced6-4f5d-b782-d70bb9a53044]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.251787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3380e14-f7c5-4525-8937-ac6c956c79e3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.274153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90d03103-d4bd-4d5d-9f0f-adcb6e2c64c3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.307617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8116111-0067-4e64-8432-0d282c7c5097]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225001, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.331638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[413a1b35-e494-4870-8861-9deaef113db7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.357955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c112dd0d-c55e-43ae-9543-0c90a5a58a90]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.384127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aee9041b-02ac-41d5-b328-1b416af40a70]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.405946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d11fd7c-bb43-4e4f-8f63-5800a114efb7]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.406816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:09.407810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:09.408305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:09.408757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:09.431354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:09.432256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96013298-b2ca-461e-8d8c-9444c9c4d357]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.437435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d111b5e-4e9c-4f7f-a9a6-21de85e2bf75]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.439825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:09.440163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 10:56:09.441183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:09.441977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:09.442575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:09.443095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:09.443720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:09.486759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0cb81c88-a7dc-407a-a856-7dd035cbb4bf]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.531576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4cb9e446-3fa2-4d8f-b098-83db700f4907]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:09.531576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:09.531576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:09.531576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:09.531576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:09.531576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:09.531576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:09.531576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:09.531576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:09.531576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:09.531576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:09.531576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:09.531576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:09.532736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cb9e446-3fa2-4d8f-b098-83db700f4907]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:09.532967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:09.535392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:09.540175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[760e04f2-748d-4d6f-ad0c-298746e23734]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.540175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:09.540175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:09.540175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:09.540175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:09.540175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8030701e-94f9-453b-9131-1b1ca99f1327]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.540175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b44dbb9-7453-4400-b86f-12cb7c571df1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.540175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d22787b5-b0ae-45d3-82c8-e4ded58b42a4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.565979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00862bc7-695b-4488-b8c2-03a012ad41a6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 786556, 'reachable_time': 23424, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225023, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.575679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:09.575941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1489546-85f0-4ad8-ae7a-d4b5d29f285c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:09.577167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 1, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 1, update_id 39878bbd-860b-48b1-b68e-741885eb03e2. Wait time elapsed: 0.720 Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 1, update_id 39878bbd-860b-48b1-b68e-741885eb03e2. Wait time elapsed: 0.720 Jul 01 10:56:09.580243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:11.186806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.608s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:11.187464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:11.188751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:11.189180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:11.189558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:11.190864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5218c74-0cf5-4588-9421-21bde297fd0f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.219406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a126f86-7a20-4207-90b7-bd604a9e5f28]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.252182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a858efc-9da5-4bf4-94c9-012b9e25a7f5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.294389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4eadd1a2-d4b9-49f8-b00c-879415ef9239]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225077, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.315659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5eaadb66-6eea-4e25-9d90-43d3c27bbb4b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.337839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f60903c4-9287-45e7-9a10-fd6472b73d78]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.360860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc9ee7ae-1f7f-410a-b958-5a60762dbbfb]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.383735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[446048fd-23f7-4c15-873a-361f3be45ca8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.384405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:11.384743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:11.385234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:11.385575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:11.412314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:11.413637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1d1a035-ffe2-4059-8b3d-fcb629617385]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.426737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a0519a1-050e-4715-ada1-262f713918ba]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.433596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:11.477377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:11.478045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:11.478445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:11.478827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:11.479201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:11.525314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a45312f-6a98-45dd-98dc-7871ce846ff6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.580324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2e312fe2-7f69-4361-b2d6-5f13c936d92c]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:11.580324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:11.580324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:11.580324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:11.580324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:11.580324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:11.580324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:11.580324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:11.580324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:11.580324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:11.580324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:11.580324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:11.580324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:11.581607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e312fe2-7f69-4361-b2d6-5f13c936d92c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:11.581873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:11.585631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:11.589881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1026277c-ceaa-4172-9eb7-851448e93547]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.589881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:11.589881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:11.589881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:11.589881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:11.589881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5025e5e4-2386-4aa9-9b19-13d135dae906]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.590484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d482f47-bde2-46e5-a494-b468699bbf64]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.591867 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d9adf2c-ea12-426d-87a5-446ef28719b2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.614919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[afe95dc0-1395-4f01-a2ce-fc6135eb3d10]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 786755, 'reachable_time': 22212, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225096, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.622896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:11.623215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acaf8434-eb7d-4dd7-8f6d-bc4cee4df811]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:11.624562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:10Z", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:11.628281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 1527f167-1f66-4d7f-9aca-85fe48351129. Wait time elapsed: 5.377 Jul 01 10:56:11.644312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 1527f167-1f66-4d7f-9aca-85fe48351129. Wait time elapsed: 5.377 Jul 01 10:56:11.644312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:14.319936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.692s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:14.320883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:14.322069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:14.322392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:14.322672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:14.324008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92f62628-253c-483f-827e-718047b2805b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.377726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d18d215-361c-4d49-8cd5-de982e3872a2]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.491091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42060920-dc51-401e-a423-699e52d40994]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.570541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c4fc346-e222-4f14-91cb-35525a3bc63a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225133, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.598747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5709c661-2056-4166-bdc6-0646ec102ffd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.635074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f5d53e3-a0cd-4237-9495-959a816afd09]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.676623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6785c5b5-0d48-4348-b11d-54f4e4b71863]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.706922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78602ea8-44f9-498f-a098-4ea52ca5f572]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.707875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:14.708366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:14.708842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:14.710053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:14.728858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:14.729058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f0291d3-9d3b-4d9b-9396-b52e107420b7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.732087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd060975-38eb-423a-866f-34d2f16d41d8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.733114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:14.770114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:14.771033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:14.771920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:14.772527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:14.773142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:14.842679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd2e98a5-24cc-44ab-a58e-c6d1844a0c04]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.901647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f13df8f9-64a3-425d-9518-1fc661fa40f8]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:14.901647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:14.901647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:14.901647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:14.901647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:14.901647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:14.901647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:14.901647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:14.901647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:14.901647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:14.901647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:14.901647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:14.901647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:14.902855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f13df8f9-64a3-425d-9518-1fc661fa40f8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:14.903583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:14.905677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:14.908416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a64628d-1e3b-4e94-9171-5671ebcff459]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.909062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:14.909737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:14.910169 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:14.910509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:14.911651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41a074a5-7f88-42cb-b484-637583c6f86c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.912873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0392aa61-ff97-46ea-893a-1c69d9a07206]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.913903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9c11c13-370c-42c2-a707-714b5f9e623f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.932669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f0e91248-f8fb-41e1-a6d5-2429c366fba9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 787083, 'reachable_time': 41096, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225165, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.938975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:14.939337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d250e01-2ceb-4297-9449-05266ca21885]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:14.940461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:10Z", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:14.944865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 39878bbd-860b-48b1-b68e-741885eb03e2. Wait time elapsed: 6.083 Jul 01 10:56:14.957351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 39878bbd-860b-48b1-b68e-741885eb03e2. Wait time elapsed: 6.084 Jul 01 10:56:14.957351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:15.165994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:15.166362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:15.166817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:15.167172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:15.355562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:56:15.634577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.691s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:15.634917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:15.635875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:15.636198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:15.636522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:15.637691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70e3962c-7b36-4210-a8f9-4d2d6b74dcf0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.651877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[befd212f-120c-432e-91f4-52b08b68d64b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.674596 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4578519e-7cd4-4b40-8dd9-61279f2fc90a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.702627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3df08f7-0eec-4cc6-acf9-02628b889a60]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225169, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.723395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3dabaf0-9c0a-4320-be8d-52a616aef530]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.744066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b04a1c2a-0637-469c-90b5-36c7c2b50ae1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.765103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1f14294-32f6-408d-937f-88780afd83c5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.785671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7ddf716-b82b-44ca-aa2c-4240428fbbcc]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.786236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:15.786508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:15.786767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:15.787040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:15.801744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:15.802039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac97c28a-8712-4eb5-ad43-59abec20ba79]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.805173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f013f602-ef5a-4361-82d2-e63a5882b8e5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.806162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:15.844213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:15.844932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:15.845525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:15.845938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:15.846505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:15.904766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[285cd01d-3467-46ae-8a18-4bba6873fb72]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.938085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b8fdf030-b760-4169-8e50-d83eb936c1fe]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:15.938085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:15.938085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:15.938085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:15.938085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:15.938085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:15.938085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:15.938085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:15.938085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:15.938085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:15.938085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:15.938085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:15.938085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:15.939458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8fdf030-b760-4169-8e50-d83eb936c1fe]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:15.940382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:15.942546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:15.947732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8dc83ad6-f9e2-45c5-b6d0-e0118cb474a5]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.948621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:15.949031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:15.949515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:15.949996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:15.951257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c40ec629-214f-4c97-bf71-86b3752c52dc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.952369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d816cd92-be23-4d45-90be-5dbec14c9839]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.953283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6271575-c1e9-49db-9c86-31f9acb2282f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.974027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[463bbc82-19ce-4553-8d58-2926dcea761d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 787196, 'reachable_time': 23776, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225188, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.981192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:15.981451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2a4c7f7-d854-493b-a9ed-4036733c0461]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:15.982459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:13Z", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:15.986016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 1527f167-1f66-4d7f-9aca-85fe48351129. Wait time elapsed: 9.736 Jul 01 10:56:15.998667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 1527f167-1f66-4d7f-9aca-85fe48351129. Wait time elapsed: 9.736 Jul 01 10:56:15.998667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:18.854865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.869s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:18.855853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:18.857272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:18.857601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:18.857880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:18.859194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a74e1880-d879-4c32-8826-7de2cbd930ca]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:18.879156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47699ae7-6f7e-4564-8772-65161e822a3e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:18.903021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f869e49f-14a0-4b04-b475-e576799f1149]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:18.936740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0cfbf25-a534-4e76-8c6b-a0bf7dcc3f58]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225196, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:18.959945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3588715b-1262-4fc5-9eda-70b79eb9854b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:18.985916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68e3c95a-267e-4cee-a50f-e3a5bba9a445]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:19.019123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5397d4d-cad1-4387-9437-ed093f73d390]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:19.045869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfc1c5f3-7ba7-473e-999c-f0efe1de98e7]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:19.046436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:19.046716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:19.046988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:19.047269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:19.071934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:19.076205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d19673a-1e46-4800-9b9d-068404935c1d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:19.079860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[903c0b8e-cc8f-4cfc-a55c-82044d4d978c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:19.088694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:19.128723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:19.129732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:19.131825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:19.132870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:19.133450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:19.223837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b34b22fc-9f1d-4bdb-82f9-f6ab312b6624]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b27d80db-12e1-46fc-9948-2afcf7dcdebf]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b27d80db-12e1-46fc-9948-2afcf7dcdebf]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:19.279430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:19.284119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:19.289852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0839c30-21dc-4f96-a877-276bf67aac26]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:19.291052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:19.291417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:19.291665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:19.292196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:19.299226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b6b0d39-edf0-464b-aada-8ad53c5ae796]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:19.300338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b9f74bc-dee6-4480-80f7-ff26345bb04e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:19.301318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e9304d0-2e6b-4478-b73f-54ffa09e1c33]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:19.360597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ab07aa7-50e1-4c6d-8a6c-5c477a13a749]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 787520, 'reachable_time': 30660, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225238, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:19.369569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:19.369882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e53469f1-b75c-4d2b-8f52-da920d686717]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:19.371001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:16Z", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:19.397413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 39878bbd-860b-48b1-b68e-741885eb03e2. Wait time elapsed: 10.521 Jul 01 10:56:19.442900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 39878bbd-860b-48b1-b68e-741885eb03e2. Wait time elapsed: 10.521 Jul 01 10:56:19.442900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:20.318590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.938s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:20.319252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:20.319830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:20.320152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:20.320436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:20.321770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[586ca434-7304-46ee-b0da-c6675d51abc6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.335305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5dcc437f-257b-4208-ad61-2a1ed8a877e3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.357731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58487e08-ed43-4dd3-97e5-2c2c0e93d8a0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.388161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3131dc1-74bf-4cb7-a0c0-6b08cc9fb3a7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225265, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.409753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7259b7a3-2277-421c-aec2-aeb97647023a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.431002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8cf8b84f-aa0f-4c4e-9a99-c1a23d86b7b7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.453349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d4412a0-d5c5-4ed2-a2a4-5b516f68fd15]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.476273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66edb7b8-d7f3-483e-8ac1-4452858cf7c5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.476874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:20.477151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:20.477550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:20.477837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:20.492752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:20.493117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41be9811-248a-4c2a-beaf-2c431c0ad852]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.497450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f6ef555-62a9-4791-bfd2-bb4ac2eabb34]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.498637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:20.540016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:20.540746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:20.541435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:20.541902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:20.542364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:20.604047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4461c822-e596-4f63-9f3f-12232361a959]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.640084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c74f55c3-f942-411d-8764-e95fe63796ed]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:20.640084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:20.640084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:20.640084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:20.640084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:20.640084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:20.640084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:20.640084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:20.640084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:20.640084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:20.640084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:20.640084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:20.640084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:20.641254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c74f55c3-f942-411d-8764-e95fe63796ed]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:20.641538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:20.643670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:20.646676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94c6619f-4912-41a2-9d56-44e271214a56]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.646676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:20.646676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:20.646676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:20.646676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:20.648015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f96e749-667a-470e-8ac8-d5a3695e9e28]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.649062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b96359d-be2b-4296-b911-927e61a881c8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.650173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db63fd63-d777-4c16-8423-3f819145a659]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.671612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ce63f4d-70b7-4533-b7ef-5b6bd67eb36e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 787665, 'reachable_time': 19428, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225283, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.678451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:20.679335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[babbaf2b-4761-4f19-a00c-17748cd13d3b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:20.680868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:18Z", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:20.684426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 1527f167-1f66-4d7f-9aca-85fe48351129. Wait time elapsed: 14.432 Jul 01 10:56:20.700371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 1527f167-1f66-4d7f-9aca-85fe48351129. Wait time elapsed: 14.433 Jul 01 10:56:20.700371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:23.111713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.429s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:23.112301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:23.113669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:23.114061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:23.114413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:23.116419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[469d3f58-1e62-44f9-b07c-c6fb2ab4fb3c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.129430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a96fd661-4b4d-4f56-a8c2-a9bfdee33c03]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.150177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3edb8e82-243a-44c3-ad10-99113aafe150]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.179825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b34f9b8-a158-4117-8ec6-edfc32206ca2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225289, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.206959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44f35a1b-5c61-45b0-b455-2ba1e0571528]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.230855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5db18b3-718e-4efd-b0e6-00d3514c2e38]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.255432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07f53936-b39e-4bae-b7b7-dee572cdcf01]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.277669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c322f9d3-ec86-4934-adef-525bd972ed9c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.278258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:23.278539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:23.279000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:23.279492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:23.295290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:23.296207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac1276ad-7a3a-42df-ba19-17dd5ba831a2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.299136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd69919a-4c6a-447a-bb06-8a7fb4eb9afe]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.300354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:23.337294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:23.338370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:23.338977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:23.339595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:23.340561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:23.409405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e51b0a3f-e6d2-44cd-ad80-e4f272bd1334]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.440190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[80012960-8d9d-49c4-88a7-213ba0319376]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:23.440190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:23.440190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:23.440190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:23.440190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:23.440190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:23.440190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:23.440190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:23.440190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:23.440190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:23.440190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:23.440190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:23.440190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:23.441229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80012960-8d9d-49c4-88a7-213ba0319376]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:23.441640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:23.443653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:23.446025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fb54f26-45f2-4949-83d5-e3c8f5670136]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.446025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:23.446025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:23.446025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:23.446025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:23.446745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7764ee76-3f95-4c5f-b0d3-8849760d21b0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.447936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e07e96f6-4964-4632-ad95-e03a06a18cea]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.448879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c7b9bc2-1b2f-4cc1-b970-a1c4c6e81a07]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.469347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27b9cc00-4b74-4e6f-a243-7b057b8f55bf]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 787944, 'reachable_time': 27052, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225307, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.476399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:23.476399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14208113-816f-48e0-b897-c7f7742bd6d3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:23.477091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:21Z", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:23.480290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 39878bbd-860b-48b1-b68e-741885eb03e2. Wait time elapsed: 14.620 Jul 01 10:56:23.494386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 39878bbd-860b-48b1-b68e-741885eb03e2. Wait time elapsed: 14.620 Jul 01 10:56:23.494386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:24.195383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.716s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:24.195983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:24.196691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:24.197075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:24.197418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:24.198615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aae29813-ace7-4255-ac5b-bb5a496aabeb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.211521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08f61a45-aabe-4807-924e-00f87ba8590e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.232570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[656ada60-484c-42ae-a36e-603611500223]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.261914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5db447c-878f-43b6-85ee-70528229bbe1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225310, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.283104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c36de988-331c-4695-91d1-33fe08027e84]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.303811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ce7ce25-c4b6-4902-adba-c40850546d58]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.325434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c7592c1-cb68-46df-a949-77204589ce7e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.346922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[431683f2-aba4-4ecd-af87-b89c47b13a6f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.347633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:24.347912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:24.348206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:24.348488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:24.366319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:24.367784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[034d8ec8-64c6-40ab-960c-d825693bb4b9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.371755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c57215aa-127e-4155-ac52-78420f6c65ae]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.373227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:24.414034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:24.415223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:24.416001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:24.417114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:24.417766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:24.490954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d5d227d-5690-4a21-89b3-334d2f543b98]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.541485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8a02bf5d-546c-4fc3-b03f-adce40060b2e]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:24.541485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:24.541485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:24.541485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:24.541485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:24.541485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:24.541485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:24.541485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:24.541485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:24.541485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:24.541485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:24.541485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:24.541485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:24.542600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8a02bf5d-546c-4fc3-b03f-adce40060b2e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:24.542791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:24.545784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:24.550659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f39d2f06-4ef3-4bf5-bb14-0b7c185cea0f]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.551901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:24.552120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:24.552540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:24.554442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:24.555786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36faa3b2-273d-4878-b08b-0e2848fee458]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.561432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cff92ec7-ece0-4593-8716-5bb5791b04da]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.562794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e09bb39e-6b2a-4633-aa54-c00c67b6fd3b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.594798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c5d1a47-e04c-410c-80d9-219267edd7b6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 788052, 'reachable_time': 19628, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225333, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.603871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:24.604382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fc1ade5-0646-479d-8b3d-b7c93ffd4d38]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:24.605649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:23Z", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:24.611058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:24.635474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 1527f167-1f66-4d7f-9aca-85fe48351129. Wait time elapsed: 18.373 Jul 01 10:56:24.635474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 1527f167-1f66-4d7f-9aca-85fe48351129. Wait time elapsed: 18.374 Jul 01 10:56:24.635474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:26.899168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-5d7dea9c-f8d7-4048-99f5-55d42b8cad62 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['72167c49-ec60-45f9-b96a-8d47b19f2308'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 10:56:27.293687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.670s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:27.294579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:27.295769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:27.296336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:27.297463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:27.299019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e23c3519-d137-4e9c-9421-8962cc1db052]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.323338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ca7dd80-a3a4-43da-a3d7-7086cfd1e60d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.353235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5121b4f-6a38-422e-85c3-65d716ab4bd5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.386020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[025d8d5b-02d7-4744-b621-7ec01a1f009b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225380, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.407062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[202aedbd-1776-4fbe-b32d-1436a92fd7db]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.433488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed46d3ec-09c9-4d26-9f5c-b2dbdd333d00]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.459134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b5f3bf6-448f-45af-b13a-6296ce21a563]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.480524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb8d0140-d662-435b-997a-f65aeddd52d1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.481267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:27.481556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:27.481808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:27.482614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:27.500962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:27.502026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd361cf0-c4a1-42f7-87e6-82080784b81b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.505989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c55569ae-4b75-46b6-b0ee-31103218e3fb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.507675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:27.556375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:27.557139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:27.557717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:27.558269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:27.576066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:27.635538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae583f6a-a4af-40a4-a64b-1a8f96c7a865]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.664607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ad5e0b06-3885-482f-8b48-b8dfe4c7b348]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:27.664607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:27.664607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:27.664607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:27.664607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:27.664607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:27.664607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:27.664607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:27.664607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:27.664607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:27.664607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:27.664607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:27.664607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:27.666367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad5e0b06-3885-482f-8b48-b8dfe4c7b348]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:27.667692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:27.669563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:27.672034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29488847-6d01-42df-a8e1-595d935c1cb7]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.672428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:27.673021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:27.673127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:27.673997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:27.675072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[616d14ea-91fc-4072-8c83-de0807168c4e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.676214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[764dd74e-b4d3-4a56-9c23-6bafc9d8651a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.677322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67e2ba79-bfa6-44ec-9bdc-d60debd5655f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.698475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d540404-305a-4249-88cc-82083153a9e3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 788364, 'reachable_time': 22712, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225398, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.705339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:27.705595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0cf12f45-5e81-4c88-889b-6deaab711303]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:27.706789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:24Z", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:27.711224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 1, update_id 862c700e-dd67-4390-bccf-61b9a80b23da. Wait time elapsed: 0.809 Jul 01 10:56:27.730998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 1, update_id 862c700e-dd67-4390-bccf-61b9a80b23da. Wait time elapsed: 0.809 Jul 01 10:56:27.730998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:28.555245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.846s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:28.556110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:28.556640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:28.557090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:28.557472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:28.558685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00782515-6645-462d-97b6-70a167e173e0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.575672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[358972be-7cb0-4a3f-b226-daf211f08813]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.597100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8ccb83f-ec4e-4546-b659-b11eb58d0d89]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.625399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65dd0a96-92f7-4568-acb3-81174ef13243]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225403, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.648773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa57eb47-eafe-4d63-89d1-e5fad06da716]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.675121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1aa8eeb2-832d-4d16-a0b0-4e50d1251950]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.696604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[320752ba-da2d-4b01-941a-152ee33a0a0b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.717264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[350be46b-1d5e-44ef-b987-8a51d854ecf0]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.717843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:28.718117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:28.718381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:28.718661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:28.733908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:28.734484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f41c589-9618-4e30-948e-4783829e264f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.737197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0f01df5-9041-443c-a59f-0e6588915748]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.738236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:28.783519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:28.784196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:28.784594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:28.784983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:28.785498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:28.846643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1642e29-f271-4937-b5a9-3a6d28787177]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.878869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[98001798-b26a-4ce6-967b-f532f7aa6136]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:28.878869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:28.878869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:28.878869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:28.878869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:28.878869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:28.878869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:28.878869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:28.878869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:28.878869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:28.878869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:28.878869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:28.878869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98001798-b26a-4ce6-967b-f532f7aa6136]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:28.880159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:28.883023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:28.886527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a5066e0-41eb-4c8d-9a73-198aa96f81a5]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.886527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:28.886527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:28.886527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:28.886527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:28.886527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7298761f-2761-4e9c-8f61-2a6bbf3a087c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.887579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[673e4f11-c621-48a9-9683-6398d4f9bdb8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.888528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d9cabed-36ff-4f40-98fd-b722b81b08dc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.909202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0167047a-2adb-4522-8ed1-39f23d0d9e04]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 788488, 'reachable_time': 21604, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225422, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.911300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:28.911764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[683fa0df-1cfb-4732-8fe0-d2b5ad1c9c23]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:28.915512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ac3dd9b1-099a-4acb-87b4-9b4358c8cbdd", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.119", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "249de301-46dc-4f85-9b7e-529715a13bd7", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6805, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bd:fa:99", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "57207112-c4f8-4080-8240-18430d7c8fea", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:26Z", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:26Z", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:28.919787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:27Z", Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 39878bbd-860b-48b1-b68e-741885eb03e2. Wait time elapsed: 20.058 Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 39878bbd-860b-48b1-b68e-741885eb03e2. Wait time elapsed: 20.058 Jul 01 10:56:28.936239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:31.885045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.968s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:31.885629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:31.886262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:31.886587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:31.886874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:31.888104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc7e8ba7-54a3-4c85-8325-10981d0fe473]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:31.901689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de56cfc4-b310-49fd-9ddd-bc55d48678b7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:31.922733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ca84adc-a4de-4dc5-bd95-ce92986aa6ad]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:31.954991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2dbba82a-ffe4-47d6-b075-12e237c22e61]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225508, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:31.977440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5d4ba83-1bdd-43b1-b699-0420218b81f5]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:31.999881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cf3a874-a791-4a3d-88c9-0e3770dc351a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.021421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45c7a0bc-964b-41bc-9d0a-638c8c332e1a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.043092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52781dd2-32d6-4356-9713-768b49bddb1e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.043679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:32.043964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:32.044234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:32.044515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:32.063043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:32.064023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d92a71ff-623c-4769-a589-90e85348c6e8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.066987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee2345ce-6044-409b-b638-87b70ae4ba0e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.068037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:32.100113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:32.100789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:32.101424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:32.101814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:32.102193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:32.160152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc4ab121-59c1-4d85-8fc7-24bc24003da9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.202606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[36810ea8-82c9-4d14-85c1-0e1280fd56c7]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:32.202606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:32.202606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:32.202606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:32.202606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:32.202606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:32.202606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:32.202606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:32.202606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:32.202606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:32.202606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:32.202606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:32.202606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:32.204157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36810ea8-82c9-4d14-85c1-0e1280fd56c7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:32.205138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:32.207414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:32.210947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a9f82fb-d61d-4e59-ad17-df8103cbba61]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.210947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:32.210947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:32.210947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:32.210947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:32.210947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a6863a3-284f-45ea-9cf3-a7418d7d74d2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.212391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8972c41-6643-4799-b6b8-d8ccadebda62]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.212951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86dd5dc7-1698-44bd-8b94-8ca954621821]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.233790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[97e096b3-8700-4b96-a217-0396c832571e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 788822, 'reachable_time': 31272, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225526, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.235932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:32.236358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4198449e-42da-4488-bea4-2a38e8b58e87]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:32.237489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ac3dd9b1-099a-4acb-87b4-9b4358c8cbdd", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.119", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "249de301-46dc-4f85-9b7e-529715a13bd7", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6805, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bd:fa:99", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "57207112-c4f8-4080-8240-18430d7c8fea", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:26Z", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:26Z", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:32.243800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:28Z", Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 1527f167-1f66-4d7f-9aca-85fe48351129. Wait time elapsed: 25.995 Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 1527f167-1f66-4d7f-9aca-85fe48351129. Wait time elapsed: 25.995 Jul 01 10:56:32.259819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:32.977316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.732s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:32.977932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:32.978703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:32.979086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:32.979450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:32.980613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[764adcbe-8d18-455a-9c74-6101015b3cad]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:32.993522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ebe61f4-0196-4f4e-b434-88e772f1e267]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.016194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abf9584f-bde6-4acf-8e84-6cdad53592e5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.046054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4d0400c-4e1e-47a9-be1e-0fe2833edfff]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225529, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.067258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ebb99b49-dda7-4efd-997c-d9f75ff71f6a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.088063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d148cb4b-8a0b-4209-8111-773ac9e91a45]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.108431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6499753f-bfbb-4650-aaf8-fb6c99508300]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.128991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81c46260-4e76-4d8c-b816-68cae40100fa]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.129556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:33.129828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:33.130094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:33.130370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:33.157733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:33.157947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f3d7784-401b-49f6-b95e-c342d24996d2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.160798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b410a66f-3f07-4027-be47-140f5b087061]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.162035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:33.202549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:33.203324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:33.203838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:33.204295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:33.204888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:33.272806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[734efa43-8c9d-4ee9-8bfc-3b3228cf1078]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.313076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c03a4c0f-c099-4298-9371-5fe3f68c215f]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:33.313076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:33.313076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:33.313076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:33.313076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:33.313076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:33.313076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:33.313076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:33.313076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:33.313076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:33.313076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:33.313076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:33.313076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:33.314338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c03a4c0f-c099-4298-9371-5fe3f68c215f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:33.314989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:33.317243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:33.321288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21a58dfa-4067-4252-9a06-d6b1b2f83686]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.321574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:33.321963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:33.322221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:33.322624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:33.323719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e6722b0-0308-4b61-b4e2-4fb4ba4bdf9f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.325234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8d31fc2f-25f7-4721-a6db-0cb20655b531]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.326279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1ec229c-66f7-4a10-b112-12a81118cd89]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.350912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0767507e-73d2-40de-a484-16ab86c61659]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 788930, 'reachable_time': 16128, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225548, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.356599 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:33.356714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a71b44ff-858b-4f49-8ec4-86ae8a6166b8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:33.358804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ac3dd9b1-099a-4acb-87b4-9b4358c8cbdd", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.119", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "249de301-46dc-4f85-9b7e-529715a13bd7", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6805, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bd:fa:99", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "57207112-c4f8-4080-8240-18430d7c8fea", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:26Z", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:26Z", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:33.363502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:32Z", Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 19, Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3 Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 862c700e-dd67-4390-bccf-61b9a80b23da. Wait time elapsed: 6.461 Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 862c700e-dd67-4390-bccf-61b9a80b23da. Wait time elapsed: 6.461 Jul 01 10:56:33.378374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:36.479375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.118s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:36.480433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:36.481695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:36.482031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:36.482316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:36.483572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88a06c13-ab1b-4bc1-b68b-b4955e9ce6df]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.502440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2cf8a926-2c82-4fa1-bf09-88680211649c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.527559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d00b7c2c-1285-4542-896e-c9b384e18676]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.566173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[554fe404-691f-4b08-b628-f6e70b654ca4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225599, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.592471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6583a43-7f61-4099-a206-09fd1e045a3d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.616370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b84146f6-1c25-4a15-8bba-3b2c82eab007]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.640088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ad33c48-8b10-4828-929b-041c30455cae]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.660948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47a0f227-e900-43b7-9cc2-ca7579558b8b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.661564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:36.662474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:36.662793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:36.663083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:36.683955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:36.684256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fca7fa1-bd40-4f77-914f-c33ae4717ac2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.687143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9bf5260c-9751-4164-83e9-cc9e2ab9d0a6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.688125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:36.723198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:36.724372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:36.725840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:36.726255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:36.727408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:36.790911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1446136b-7be4-4dc1-ba46-668b8096cdd4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.821828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[720d66a5-2a3c-4f1e-9334-e73539a3799d]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:36.821828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:36.821828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:36.821828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:36.821828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:36.821828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:36.821828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:36.821828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:36.821828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:36.821828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:36.821828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:36.821828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:36.821828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[720d66a5-2a3c-4f1e-9334-e73539a3799d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:36.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:36.825536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:36.828491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7fddc5f5-74da-4618-b729-fd07bf17c46d]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.828908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:36.829294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:36.829559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:36.829955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:36.831021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b1c1ec3-9ba0-46c6-b49d-64273c92d6b5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.832069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ce09613-bb22-4f1e-bc70-d9f43c18f0be]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.833001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f012759d-5f5c-44ad-92ae-91e9407cc335]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.853394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee3f1265-dbe8-49f2-b53a-614a51e4e9dc]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 789282, 'reachable_time': 16796, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225619, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.860396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:36.861062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f4601b6-bb9f-4508-bb6f-471b5d2c2e4c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:36.862093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ac3dd9b1-099a-4acb-87b4-9b4358c8cbdd", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.119", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "249de301-46dc-4f85-9b7e-529715a13bd7", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6805, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bd:fa:99", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "57207112-c4f8-4080-8240-18430d7c8fea", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:26Z", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:26Z", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:36.866972 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:33Z", Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 39878bbd-860b-48b1-b68e-741885eb03e2. Wait time elapsed: 28.022 Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 39878bbd-860b-48b1-b68e-741885eb03e2. Wait time elapsed: 28.023 Jul 01 10:56:36.884587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:37.689030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.807s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:37.689720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:37.690223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:37.690546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:37.690832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:37.692041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00aaa362-7c53-4f94-b549-aeac36c34cbd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:37.705400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f7044443-faa1-4be8-85ed-79454d66979a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:37.728380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0704401a-06a0-43fe-924a-b4b3b8fa6d6d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:37.758754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f449708-a28a-420c-ae10-d189ae7c02dd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225622, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:37.779796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e791a990-6765-432e-baf6-ccbb7e03afe4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:37.800704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb06827f-5b8f-4ace-91a7-9220e7e35741]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:37.821674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c79ef7c-c801-404e-b3fe-84149232609d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:37.842000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f173589a-7942-48e3-8f69-2ff01ecc0a2d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:37.842646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:37.842983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:37.843312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:37.843702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:37.859038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:37.859822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96de58d0-9349-4e6c-8540-a9f30b7abd80]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:37.862592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4f5c9d3-a56b-414a-967f-dc39fe576619]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:37.863640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:37.904207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:37.905432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:37.906320 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:37.906944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:37.907337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:37.983518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35654431-63e6-4bce-9163-936e5f6874ca]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:38.031026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b1f0668a-fba8-44e9-98d5-82ebdcc68536]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:38.031026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:38.031026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:38.031026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:38.031026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:38.031026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:38.031026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:38.031026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:38.031026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:38.031026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:38.031026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:38.031026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:38.031026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:38.032797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b1f0668a-fba8-44e9-98d5-82ebdcc68536]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:38.034100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:38.036046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:38.038843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6858027e-8df2-4f80-8367-d0f2ab32b1bd]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:38.038843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:38.038843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:38.038843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:38.038843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:38.040736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43440cd3-12ba-41c4-9dc5-b67f1bb543dd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:38.041967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4597c05-9e5d-4741-87e5-ca1723112095]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:38.043185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8a97447-fa17-4eb5-bd1e-5b5d5cb43db9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:38.066378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d282637-f446-4cad-9682-a30fddac1f85]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 789402, 'reachable_time': 32932, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225641, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:38.074890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:38.075234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5dfec6db-a692-46a5-a72d-f69b39d58df5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:38.076446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ac3dd9b1-099a-4acb-87b4-9b4358c8cbdd", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.119", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "249de301-46dc-4f85-9b7e-529715a13bd7", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6805, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bd:fa:99", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "57207112-c4f8-4080-8240-18430d7c8fea", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:26Z", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:26Z", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:38.079627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:36Z", Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 23, Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3 Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 862c700e-dd67-4390-bccf-61b9a80b23da. Wait time elapsed: 11.178 Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 862c700e-dd67-4390-bccf-61b9a80b23da. Wait time elapsed: 11.179 Jul 01 10:56:38.093719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:41.160247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.082s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:41.160848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:41.161475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:41.161798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:41.163377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:41.165141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[907df5b0-f53d-49d1-9ae7-1d58d5c4bba7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.184513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1a354fa-9605-44e6-b0fb-b618a49870db]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.206434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24e893d7-9498-4fce-9740-fce861d7f31f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.237594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6b5b272-1892-491e-8487-124a5ca872a0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225695, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.271538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bc13464-96bd-49bf-bc5b-2c744da44b5a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.291926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e678bff-517a-4f5e-bd33-3144f895dd7b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.312677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14c1f1b3-7dde-439d-affd-a50073cdf5ad]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.333392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cd5005a-1389-4288-81ba-5d4e8a7007f5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.334211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:41.334554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:41.334896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:41.335183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:41.352439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:41.352738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45e919f0-9340-4359-b86e-1c73c2f97a1a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.355623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[244ce6d9-0174-4931-9b4d-4178c28aaca7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.356655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:41.394830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:41.395544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:41.396849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:41.396980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:41.397666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:41.454392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[078a114e-279c-4cd5-86cb-35b1dbcca97e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.491697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[046cd10c-46f9-4119-a8d4-38a08e68ccad]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:41.491697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:41.491697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:41.491697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:41.491697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:41.491697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:41.491697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:41.491697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:41.491697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:41.491697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:41.491697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:41.491697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:41.491697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[046cd10c-46f9-4119-a8d4-38a08e68ccad]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:41.492920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:41.494796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:41.497519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b6a05c5-1d9a-4f72-a25d-91c55095d06f]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.497519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:41.497519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:41.497519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:41.497519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:41.498035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70c9cc61-546f-4836-bd56-7aa8f7ff42c4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.498977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a417f2c1-5b5f-4701-bb26-7fa2d2ad71fc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.499952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5271e458-dda6-40d3-a265-13a7f170d0a4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.522137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6575494-7e1e-4014-aff2-869f3794e7d1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 789750, 'reachable_time': 25212, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225713, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0412671b-9d51-48a0-a552-fa876abe423c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:41.526514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ac3dd9b1-099a-4acb-87b4-9b4358c8cbdd", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.119", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "249de301-46dc-4f85-9b7e-529715a13bd7", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6805, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bd:fa:99", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "57207112-c4f8-4080-8240-18430d7c8fea", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:26Z", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:26Z", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:38Z", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 24, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:41.531633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:41.549310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:41.549310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:41.549310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:41.549310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:41.549310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:41.549310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:41.549310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:41.549310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:41.549310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:41.549310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:41.549310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 862c700e-dd67-4390-bccf-61b9a80b23da. Wait time elapsed: 14.627 Jul 01 10:56:41.549310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 862c700e-dd67-4390-bccf-61b9a80b23da. Wait time elapsed: 14.627 Jul 01 10:56:41.549310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:42.280457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.753s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:42.281521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:42.282764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:42.283474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:42.283993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:42.285480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af7e60a1-efec-46ff-9c0d-2009f2ecc4cd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.300809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c6ecf6c-db39-4f87-b017-d9bc388150a4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.324916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bea7cdf-fc00-44ff-816d-e02c429dbcf9]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.355290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[585248fb-dc2f-4aaa-a923-6c4ff0555fe7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225716, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.377106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6276186e-9dfa-4d9e-9508-7af6c3a961fd]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.398625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27543204-12d9-4896-a17b-19af978693cb]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.419345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b16d37db-783d-4f4c-b147-09c63bac8dc5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.442690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5de5819f-18ef-4d8f-8e4a-95cd3c8960c1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.444392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:42.445124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:42.445426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:42.445937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:42.462758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:42.463111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[afa89890-cc92-41af-8f58-bd1ec9f60c15]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.465878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9f22da19-5b31-4521-a830-8d357f13f229]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.466778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:42.509067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:42.510183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:42.512153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:42.512749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:42.513202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:42.566605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f503a345-9759-4689-b3d2-ddb4c458a034]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.598415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[2836713b-eec2-4474-be27-2714d70b4741]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:42.598415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:42.598415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:42.598415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:42.598415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:42.598415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:42.598415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:42.598415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:42.598415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:42.598415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:42.598415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:42.598415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:42.598415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:42.600138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2836713b-eec2-4474-be27-2714d70b4741]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:42.600970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:42.602799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:42.605219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[76f8dba1-ee3f-461c-b7d8-4a7fc61b7599]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.605219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:42.605219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:42.605219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:42.605219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:42.605876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94880190-ad0e-4d1b-8f53-f4203f537d23]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.606840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6554d590-e7f9-45fb-b511-0ee2f9edb0d1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.607859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1332b879-2556-415d-956c-fdc205fd3b76]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.628448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94c8774b-e6b7-4be3-a5f9-04d1d5e61deb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 789862, 'reachable_time': 30780, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225735, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7010e226-e756-4601-8804-4a8a6f9d9b70]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:42.635060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ac3dd9b1-099a-4acb-87b4-9b4358c8cbdd", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.119", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "249de301-46dc-4f85-9b7e-529715a13bd7", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6805, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bd:fa:99", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "57207112-c4f8-4080-8240-18430d7c8fea", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:26Z", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:26Z", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:41Z", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 27, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:42.640608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 862c700e-dd67-4390-bccf-61b9a80b23da. Wait time elapsed: 15.734 Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 862c700e-dd67-4390-bccf-61b9a80b23da. Wait time elapsed: 15.734 Jul 01 10:56:42.654736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:45.636072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.001s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:45.636700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:45.637728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:45.638077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:45.638356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:45.639653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53f5043a-5e40-4768-bea7-68cf09b9a221]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.653085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aaed3a90-ecf7-4595-86ea-66b0e511ef69]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.675752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2716a514-feb5-45ff-bdf0-a35b7c4045e3]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.706049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1bacc8df-4feb-4ade-9995-a5f8a5e614d0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225786, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.727812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36cee5b8-7b54-42c1-bf00-cb3db41ad486]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.747874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[686ca1bd-197a-45e5-b47f-ce1b330d2e4f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.768233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37132389-6f67-4661-886c-cf003589da2a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.789117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17caf1d5-56ff-4e6a-983b-8cf9ce0921b3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.789743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:45.790088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:45.790416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:45.790756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:45.806087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:45.806788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a20939ee-f7d4-456d-81c4-c89a0dbbc7d5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.809509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe85118c-1e20-4658-ae0c-13614c7a8149]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.810492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:45.843983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:45.845342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:45.845742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:45.846132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:45.846514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:45.918627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34256285-706f-4538-9c43-29422f6ca5ec]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.956863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bf3cf65d-e555-4c6f-ae14-2a0399aee920]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:45.956863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:45.956863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:45.956863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:45.956863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:45.956863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:45.956863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:45.956863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:45.956863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:45.956863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:45.956863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:45.956863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:45.956863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf3cf65d-e555-4c6f-ae14-2a0399aee920]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:45.957981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:45.960825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:45.964227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5ab6581-26c4-4442-a1ef-aa5e5c922852]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.964227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:45.964227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:45.964639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:45.964740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:45.965831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bf23f4e-bdea-4bf1-84f3-cbf353da011a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.966939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[deaaf043-e752-4d19-b054-338399d1aeee]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.968009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12397f12-6c91-4591-a05a-1a55c20acb96]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.989016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62e42c4d-c2a0-4722-b1d5-90e1c8d5deb9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 790196, 'reachable_time': 28756, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225804, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.994412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:45.994412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c1b19b2-bec4-4165-b760-8aa1a8b60432]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:45.995519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ac3dd9b1-099a-4acb-87b4-9b4358c8cbdd", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.119", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "249de301-46dc-4f85-9b7e-529715a13bd7", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6805, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bd:fa:99", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "57207112-c4f8-4080-8240-18430d7c8fea", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:26Z", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:26Z", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:46.000646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:44Z", Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 29, Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 862c700e-dd67-4390-bccf-61b9a80b23da. Wait time elapsed: 19.097 Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id 862c700e-dd67-4390-bccf-61b9a80b23da. Wait time elapsed: 19.098 Jul 01 10:56:46.014557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 10:56:46.853415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.856s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 10:56:46.854025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 10:56:46.854594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 10:56:46.854914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:46.855186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:46.856704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36d53c6e-29de-47ac-ad3f-9cfdd680d072]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:46.872175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75f3dd2c-d23a-4055-8495-a3c22bc72a8a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:46.899101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6041132-cc4b-4b19-a26d-b45aec580059]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:46.928257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75657477-ee8f-4b6a-a1b1-c9c7fd5a40da]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 225809, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:46.956748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01db0d50-e01b-4218-bf16-2778f9f82e0b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:46.984038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ad2ae90-5bc9-4d97-873f-3e88d34f9f2f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:47.008501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b401bae-d734-4a4d-82a1-fa4d07f7bf0d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:47.031033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3d7aafd-aa02-4543-9959-16eaf531edf5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:47.031662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:56:47.031970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:56:47.032258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 10:56:47.032576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 10:56:47.050056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 10:56:47.050750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[599d96e4-a1e0-42bc-8f9e-5570e097408a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:47.053998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91760a73-3a04-4ac0-b944-fe6e9d39eafe]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:47.055251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:47.093542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:47.094206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:47.094607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:47.094998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:47.102232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 10:56:47.152843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[180565c1-3a79-4b74-822b-1eeb216ec058]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:47.185700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[24b7b8ba-c20d-432d-81d5-83a0817b25bf]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 10:56:47.185700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 10:56:47.185700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 10:56:47.185700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 10:56:47.185700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 10:56:47.185700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 10:56:47.185700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 10:56:47.185700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 10:56:47.185700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 10:56:47.185700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 10:56:47.185700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 10:56:47.185700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 10:56:47.185700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:47.186890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24b7b8ba-c20d-432d-81d5-83a0817b25bf]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:47.187497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:47.189459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:47.191988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28878290-57cc-4cdd-a496-9b9b09cc42bc]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:47.191988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:47.191988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:47.191988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 10:56:47.191988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 10:56:47.192437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9b5be55-df9f-4a9f-b069-9224a6855416]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:47.193950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32942aea-5ef1-4fd8-8cfd-40f2dc7f558d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:47.195508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd82ee23-833d-4ba5-9d22-6088fc4bd89d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:47.215470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b3635dd-0a42-4a8e-a1a2-236dbcaf8872]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 790319, 'reachable_time': 24936, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 225827, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7362ca08-ff5e-4a64-8205-2a6a41f61400]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:47.221578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "ac3dd9b1-099a-4acb-87b4-9b4358c8cbdd", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.119", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "249de301-46dc-4f85-9b7e-529715a13bd7", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.8", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6805, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:bd:fa:99", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "57207112-c4f8-4080-8240-18430d7c8fea", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:26Z", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:26Z", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:46Z", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 31, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 10:56:47.227898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 10:56:47.241937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:47.241937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 10:56:47.241937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 10:56:47.241937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 10:56:47.241937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 10:56:47.241937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 10:56:47.241937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 10:56:47.241937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 10:56:47.241937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 10:56:47.241937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 10:56:47.241937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 10:56:47.241937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3 Jul 01 10:56:55.357261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:57:15.167175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:57:15.167905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:57:15.168145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:57:15.168565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:57:35.358773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:58:15.167007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:58:15.168228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:58:15.168228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:58:15.168228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:58:15.360365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:58:55.361771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 10:59:15.167629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:59:15.168368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:59:15.168518 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 10:59:15.168693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 10:59:35.362968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:00:15.168379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:00:15.169077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:00:15.169077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:00:15.169290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:00:15.365267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:00:55.367917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:01:15.168735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:15.169386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:15.169386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:15.169519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:28.939671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-138425b4-cebd-44ba-91b0-b267c37ecd1f tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['72167c49-ec60-45f9-b96a-8d47b19f2308'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:01:28.940792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 1, update_id b341cf32-a4ec-4a76-b19b-ec9f7ae72f63. Wait time elapsed: 0.001 Jul 01 11:01:28.941223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 1, update_id b341cf32-a4ec-4a76-b19b-ec9f7ae72f63. Wait time elapsed: 0.001 Jul 01 11:01:28.941536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:31.101226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.160s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:31.101846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:01:31.102911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:01:31.103340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:31.103745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:31.105139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46b8f45f-7c3f-459d-a666-4f11af770cfa]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.127605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0134f2f-bb7d-45e9-a774-25945d931e82]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.150854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff51ccd7-8491-4e70-be57-e38f79b4e9ed]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.181040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bafdb391-fe1e-4599-9d6f-b1e86ac47491]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230262, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.204586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8b77ac8-8f2f-4c8e-b1f9-99be420917c4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.226910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aac3488c-f4e0-4c8c-83b6-399cb71b1638]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.250303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecfa12d5-6bb3-43b3-93ff-f58fccd9077b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.283999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ffd9879-0336-49c6-84d3-4bcc455feda4]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.284710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:31.285269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:31.285429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:01:31.285805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:01:31.305264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:01:31.305709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[04470a6e-80c1-4191-8cfc-cf050c52e1a3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.308959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df4d5e65-e6bf-407a-ad3f-9383ec94c745]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.310494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:31.349356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:31.350057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:31.350512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:31.350910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:31.351298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:31.417676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[223a04a3-f336-4166-8534-17474b6b9eca]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.457037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[fb927d93-1bcf-449d-a1ba-7b12fc6db98f]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:01:31.457037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:01:31.457037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:01:31.457037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:01:31.457037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:01:31.457037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:01:31.457037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:01:31.457037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:01:31.457037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:01:31.457037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:01:31.457037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:01:31.457037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:01:31.457037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:31.457851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb927d93-1bcf-449d-a1ba-7b12fc6db98f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:31.458896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:31.460359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:31.462092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9e66394-7266-4bf9-8693-54802b97ff77]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.462092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:31.462092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:31.462318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:01:31.465618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:31.466646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ca36e42-0ce4-4806-af9d-5087c34b4229]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.467733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb72a22b-6845-45dd-9bc8-a9336bce51f1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.468786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b083a7e6-0966-4b5f-a416-4a00fd507ac6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.492428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b4e510e-b9f1-4616-bee1-b11818c04804]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 818744, 'reachable_time': 43308, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230281, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.503608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:01:31.503870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[edfb04a4-619c-4886-b91f-e9ad972462f1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:31.505162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:49Z", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 35, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:01:31.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id b341cf32-a4ec-4a76-b19b-ec9f7ae72f63. Wait time elapsed: 2.567 Jul 01 11:01:31.520261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id b341cf32-a4ec-4a76-b19b-ec9f7ae72f63. Wait time elapsed: 2.567 Jul 01 11:01:31.520261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:32.346156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.838s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:32.346740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:01:32.347510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:01:32.347842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:32.348126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:32.351765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf308f4d-24be-4456-a0b9-816f1bbd476d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.372414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70bcbb58-48a3-43cd-8d0c-f2e92fbed01c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.402557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f613bf2c-010e-442c-a145-77a85bcfc1c8]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.454861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70af56a5-f1b3-44ff-8408-1a9dea1fe843]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230317, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.476261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18e6f718-66a6-42fd-bc3b-dea3315cc0ff]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.499688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df410710-d994-4e9b-8e22-9be7d4a01209]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.522307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4711fa8d-1ea9-4403-a39a-6d38d8956b03]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.542968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e941fa26-d604-4d0f-ac58-0bd73e33119a]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.543665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:32.544179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:32.544581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:01:32.544992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(b0c88eae-ae6c-4556-a0e9-a6cbb5f00589) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:01:32.562468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:01:32.562935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c1e6a96-c561-4c8b-9322-ac45de2cc27a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.565999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5808713a-ebd0-46fe-8993-db5759278db9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.567175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-b0c88eae-ae, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:32.611217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-b0c88eae-ae, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:32.612365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:32.612891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:32.613283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-b0c88eae-ae, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:32.613673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-b0c88eae-ae, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'b0c88eae-ae6c-4556-a0e9-a6cbb5f00589', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:22:6f'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:32.677637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6f3531d-5c77-4293-8609-67ec028f4c29]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.721799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[dd57fe34-8782-4928-ac3f-36e43def4f77]: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:01:32.721799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:01:32.721799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:01:32.721799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:01:32.721799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:01:32.721799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:01:32.721799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:01:32.721799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:01:32.721799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:01:32.721799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:01:32.721799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:01:32.721799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:01:32.721799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:32.722876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd57fe34-8782-4928-ac3f-36e43def4f77]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:32.723880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:32.725648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:32.729332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70521fad-091d-4a59-8a85-a50576ba5fb5]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.730136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:32.730458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:32.730711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:01:32.731054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:32.732370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af51fa3e-428b-4975-abef-58e0b2e86c47]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.733368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25c79fa7-4d19-49f1-8e5f-33da8be1b158]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.734288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01305888-d25d-4783-9768-29c7a47b5e5d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.755877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66c05266-d924-410c-8bcb-537d85c8de08]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 818871, 'reachable_time': 36040, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230338, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21f915e2-7c90-44a1-af43-88e47281e244]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-b0c88eae-ae not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 11:01:32.763500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:08Z", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "b0c88eae-ae6c-4556-a0e9-a6cbb5f00589", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "afb2987f-4d7f-43f9-b357-669b02318b60", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:22:6f", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6790, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:07Z", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:49Z", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 35, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "56917e33-12a9-4d5b-9542-c6e3bdb666a9", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id b341cf32-a4ec-4a76-b19b-ec9f7ae72f63. Wait time elapsed: 3.821 Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 2, update_id b341cf32-a4ec-4a76-b19b-ec9f7ae72f63. Wait time elapsed: 3.821 Jul 01 11:01:32.768748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:34.416854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-435a0991-34a6-4dc9-ab59-6bc4e9fefa46 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got routers updated notification :['72167c49-ec60-45f9-b96a-8d47b19f2308'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:01:35.369318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:01:35.381177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.620s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:35.381456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 72167c49-ec60-45f9-b96a-8d47b19f2308 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:01:35.382423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (72167c49-ec60-45f9-b96a-8d47b19f2308), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:01:35.382755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:35.383032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:35.384381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d53f5505-ac54-484d-91a8-5043ea8b4101]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.401312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f490247-3633-4e56-aae5-83746cc16c15]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.426779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea403e77-af0c-49f6-8102-282ef82b3c7e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.467375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41bfd013-3250-49d3-96a9-1b205aa9cca4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230390, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.488879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[498d912c-8bc5-44c8-8d57-79dc81f1e455]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.512667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3ea26a9-c314-43f8-a357-32825c6c8cd9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.534369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0da81c8-a90c-4d00-9733-f46c91162b12]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.557153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1e5d6dd-c248-4b3f-b636-bbf27dd2b857]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.558765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:35.560347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:35.560642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:01:35.585585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[733dd3cc-bf2a-4c1b-992c-2f36e85b1f7d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 819172, 'reachable_time': 34820, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230396, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.586487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:35.586829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:35.587334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:01:35.587715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 11:01:35.588111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:35.588438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:35.588793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '7d9d282d-97a6-4cc8-a937-fe2fd4c0773c', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:a1:e0:ab', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '72167c49-ec60-45f9-b96a-8d47b19f2308', 'device_owner': 'network:router_gateway', 'standard_attr_id': 6788, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.83', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T10:56:04Z', 'updated_at': '2022-07-01T10:56:07Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-7d9d282d-97), ns(qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 11:01:35.607640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-7d9d282d-97 not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:01:35.607936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f883f8f0-01a9-46b5-937b-2c67c30d6c11]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.612653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87043190-fd07-4ed3-a256-fd220a9a449f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.614214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-7d9d282d-97, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:35.614571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 11:01:35.615441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-7d9d282d-97, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:35.616060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-7d9d282d-97, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:35.616457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-7d9d282d-97, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:35.616850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-7d9d282d-97, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:35.617241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-7d9d282d-97, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7d9d282d-97a6-4cc8-a937-fe2fd4c0773c', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:a1:e0:ab'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:35.658392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73f99010-052b-4b31-9901-fe287988a849]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.713299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e0e6a1c0-dadb-4833-9861-6710760e77e5]: Network interface qg-7d9d282d-97 not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:01:35.713299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:01:35.713299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:01:35.713299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:01:35.713299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:01:35.713299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:01:35.713299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:01:35.713299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:01:35.713299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:01:35.713299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:01:35.713299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:01:35.713299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:01:35.713299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7d9d282d-97 not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:35.714137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0e6a1c0-dadb-4833-9861-6710760e77e5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-7d9d282d-97 not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.714137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:01:35.714685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 11:01:35.731950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[336f888c-88b8-4dac-9856-e2b678ba40e6]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.930630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05cac456-eb99-4cda-9ec0-9abf8e9da012]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:35.942883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[769b7706-d7e2-4b91-8a08-5f5a07892bda]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:36.134377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59afe065-e240-4351-b687-23ec9a9b8119]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:36.135123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 11:01:36.135574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:01:36.136059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:01:36.136439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308" released by "process_external" :: held 0.549s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-7d9d282d-97 not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7d9d282d-97 not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7d9d282d-97 not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:36.137381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7d9d282d-97 not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7d9d282d-97 not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:36.140388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:36.144949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[722f44a1-dd42-4822-938b-48380f6f3323]: (4, ['qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:36.146748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:36.147163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:36.147745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:01:36.148317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/72167c49-ec60-45f9-b96a-8d47b19f2308.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:36.149643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6167d8a8-2a64-4fee-b27b-213f8b3281d3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:36.150545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[964ca2db-a4ac-4b65-bef2-3136c0c5d099]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:36.151505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6de0f30e-993b-4153-91ad-0f2d723edcc6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:36.175581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8124da3-23af-42cf-bfb4-3cf4aea71dd5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 819172, 'reachable_time': 34820, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230413, 'error': None, 'target': 'qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:36.187667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:01:36.187958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c50a2fa5-58a0-4475-8ad2-6d1ffb246840]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 72167c49-ec60-45f9-b96a-8d47b19f2308: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7d9d282d-97 not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-7d9d282d-97 not found in namespace qrouter-72167c49-ec60-45f9-b96a-8d47b19f2308. Jul 01 11:01:36.189284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestNetworkAdvancedServerOps-router-700588206", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83" Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:33Z", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "71f7a01415214b618eaf4f9ed3e51c3a", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7d9d282d-97a6-4cc8-a937-fe2fd4c0773c", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:a1:e0:ab", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "72167c49-ec60-45f9-b96a-8d47b19f2308", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 6788, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.83", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T10:56:04Z", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T10:56:07Z", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 1, update_id 56d3396c-4ac2-4156-911e-e19bee7933e9. Wait time elapsed: 1.772 Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 3, priority 1, update_id 56d3396c-4ac2-4156-911e-e19bee7933e9. Wait time elapsed: 1.773 Jul 01 11:01:36.192262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:36.208896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ad00c8d2-2f3a-4974-8d63-1e21cab99f96 tempest-TestNetworkAdvancedServerOps-1462301354 tempest-TestNetworkAdvancedServerOps-1462301354-project] Got router deleted notification for 72167c49-ec60-45f9-b96a-8d47b19f2308 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 11:01:36.228572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.038s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:36.228845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 72167c49-ec60-45f9-b96a-8d47b19f2308 was not found. Performing router cleanup Jul 01 11:01:36.230881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70dc2af6-bcbc-494f-9c06-b5ca95e8b5ff]: (4, ['qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:36.231537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:36.231993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:36.232272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 72167c49-ec60-45f9-b96a-8d47b19f2308, update_id 56d3396c-4ac2-4156-911e-e19bee7933e9. Time elapsed: 0.042 Jul 01 11:01:36.232690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 72167c49-ec60-45f9-b96a-8d47b19f2308, action 1, priority 1, update_id b6705d61-b4eb-4124-820b-77416ecc5c7e. Wait time elapsed: 0.023 Jul 01 11:01:36.232927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 72167c49-ec60-45f9-b96a-8d47b19f2308, action 1, priority 1, update_id b6705d61-b4eb-4124-820b-77416ecc5c7e. Wait time elapsed: 0.024 Jul 01 11:01:36.233165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 72167c49-ec60-45f9-b96a-8d47b19f2308 was not found. Performing router cleanup Jul 01 11:01:36.234295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6851c4de-99e8-4bac-b96d-b04d1aee0bce]: (4, ['qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:36.234826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:36.235107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:36.235436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 72167c49-ec60-45f9-b96a-8d47b19f2308, update_id b6705d61-b4eb-4124-820b-77416ecc5c7e. Time elapsed: 0.003 Jul 01 11:01:41.212709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c801d76c-4e7d-4cb4-b9ab-48fb6c730651 admin admin] Got routers updated notification :['615b890e-df8b-448d-89b4-03af0d799292'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:01:41.214044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id 852755d3-d750-48f8-b06a-3c6e403e3fb3. Wait time elapsed: 0.000 Jul 01 11:01:41.214532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id 852755d3-d750-48f8-b06a-3c6e403e3fb3. Wait time elapsed: 0.001 Jul 01 11:01:41.215085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:43.347301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-dd246ebd-ce8f-4481-ada1-b306f86a02dd admin admin] Got routers updated notification :['615b890e-df8b-448d-89b4-03af0d799292'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:01:44.673674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.458s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:44.674614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:01:44.675720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:01:44.676146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:44.676620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:44.677934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2562225b-2482-4280-912c-e8cf93bf6746]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:44.694177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a67e591-98d6-4076-965e-3255e6bfb122]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:44.715019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d5a8e3d-528b-412e-9837-ae3b98bc9a5a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:44.753535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4480808-f318-4a62-8629-ab859a9163c1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230532, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:44.775111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69dbd8ea-597a-4e9b-8426-60bc6079c96f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:44.799106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7db5ea9a-7efa-4440-9446-256f3d9a2fe9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:44.821193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d6b38fa-da83-4868-8d50-a5222ea987af]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:44.843004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d05d272d-dd98-491c-a572-92a00cb3d4eb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:44.843812 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:44.844094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:44.845074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:01:44.845379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:01:44.863799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:01:44.864268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35f0c0d1-fcb0-413e-af86-144526c26b2d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:44.869495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2861e216-106c-4f64-a086-3938562fe367]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:44.870953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:44.871384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 11:01:44.872308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:44.873086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:44.874441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:44.874441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:44.874441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:44.930921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7160dd4-030d-45df-9d72-3534ab0bcf48]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:44.989206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e9aac36f-40d7-4fc6-9d06-c6b86c19fddb]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:01:44.989206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:01:44.989206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:01:44.989206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:01:44.989206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:01:44.989206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:01:44.989206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:01:44.989206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:01:44.989206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:01:44.989206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:01:44.989206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:01:44.989206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:01:44.989206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9aac36f-40d7-4fc6-9d06-c6b86c19fddb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:44.990986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:44.994333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:44.998441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2595df8-f2d3-4e7b-b2fa-972e4793a994]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:44.998441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:44.998441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:44.998441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:01:44.998441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:45.000907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25e53c94-78d0-4f36-80be-4e4b4df469dd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:45.001603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa2e0917-09d5-4a57-abac-c68cdfc08876]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:45.002652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3463e8fb-7314-458b-ad89-2977146901d9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:45.028794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eebcd667-9a50-42b3-ae9f-37c10d19c018]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 820101, 'reachable_time': 32532, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230550, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:45.036983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:01:45.037239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b25015f8-0002-436d-a249-4f9fc2de0741]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:45.038609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 2, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id 3694d684-bc77-489e-8363-e652f81c0c47. Wait time elapsed: 1.692 Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id 3694d684-bc77-489e-8363-e652f81c0c47. Wait time elapsed: 1.692 Jul 01 11:01:45.041113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:46.142979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-14f56551-790e-4272-b2f3-a71e7add5281 admin admin] Got routers updated notification :['391e582a-f7fd-476c-8bcf-9c33d2f0da0c'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:01:46.145453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 1, update_id 90e90923-6136-48fd-91e6-17f8ed3b2173. Wait time elapsed: 0.001 Jul 01 11:01:46.145958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 1, update_id 90e90923-6136-48fd-91e6-17f8ed3b2173. Wait time elapsed: 0.002 Jul 01 11:01:46.146528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:46.914322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.873s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:46.914827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:01:46.916558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:01:46.916945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:46.917434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:46.919572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49e2e6ae-e1be-4a6a-bbb7-68bd03ab820d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:46.936304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[831d71ed-d3f7-47f9-823e-598b1bf82066]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:46.958945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2239e6f-74d0-459f-b595-d66f2055624a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:46.991967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9458d72f-ad0d-4cac-916b-179bb4c89cea]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230554, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.015140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1b97bd4-8c1e-4bd2-af8d-e09d026678e2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.040231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e147ea54-fe81-4660-9878-6fa0b987c332]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.053121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d8e824f-54fe-4a2f-bc65-2bd1749cdee2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.076548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87df0962-3bee-43a9-b850-65a2d7c616c7]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.077107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:47.077542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:47.077999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:01:47.078287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:01:47.096627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:01:47.098667 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c2f9e549-2d02-4215-8973-06a572db22eb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.102662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd71f45e-7f7f-4d03-96d9-dd4d81a168dd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.103714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:47.133614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:47.135041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:47.135607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:47.136618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:47.137148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:47.214105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b95c5eed-bf80-45da-8801-95716a286675]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.254797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[eee7bf96-bb2b-4637-830d-6aeab0fb4672]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:01:47.254797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:01:47.254797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:01:47.254797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:01:47.254797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:01:47.254797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:01:47.254797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:01:47.254797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:01:47.254797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:01:47.254797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:01:47.254797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:01:47.254797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:01:47.254797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:47.256180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eee7bf96-bb2b-4637-830d-6aeab0fb4672]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:47.256773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:47.258450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:47.260609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88700453-aedd-4dde-b947-f53a291bf12a]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.260609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:47.260609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:47.260609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:01:47.260609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:47.261570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fb988aeb-c9b0-4676-a3a4-9dcc24043ff0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.262622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[388a7eef-32bd-4364-a236-55b506379a0a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.264477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30f5efbd-7eb7-413c-87db-9a66911734cb]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.287689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19a3e9a8-c3dd-479f-936d-74659a9e3008]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 820325, 'reachable_time': 35556, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230573, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.295422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:01:47.295899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c5cdfbe-5cfb-488f-a3a3-a8509d95a17e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:47.297006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:46Z", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:01:47.299472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 852755d3-d750-48f8-b06a-3c6e403e3fb3. Wait time elapsed: 6.085 Jul 01 11:01:47.316443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 852755d3-d750-48f8-b06a-3c6e403e3fb3. Wait time elapsed: 6.085 Jul 01 11:01:47.316443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:48.370502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.072s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:48.371041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:01:48.372201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:01:48.372601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:48.372942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:48.374206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b59f7547-648c-4465-bdab-0d7212a828ce]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.407698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c2d210c-b7a8-45cc-9be3-36485d9e2c2c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.449657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4772110f-d492-410b-8f53-efc401a2ee36]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.536249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cabcb20d-465b-493b-9638-451387f4d551]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230632, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.566089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9dde6334-843c-4b55-a01d-ee203c21535d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.611901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6f17c09-0a39-4c5e-842a-6a4aa8c123f5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.645963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fcec983-f40c-4674-acdb-313604d6d831]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.670306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e70933f3-9488-4a30-8bfb-6a342913ade7]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.671180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:48.671488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:48.671757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:01:48.672054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:01:48.690625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:01:48.691073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b948489-a5e9-4053-91d1-b63cad14bd14]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.694683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1dd2ee7-414e-41ea-a181-ee5ed2a2cf1c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.696029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:48.715843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-13d45a13-7bd6-4c19-851e-aaa923eae1ce admin admin] Got routers updated notification :['391e582a-f7fd-476c-8bcf-9c33d2f0da0c'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:01:48.738479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:48.739528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:48.740842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:48.741421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:48.742554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:48.815517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[368a4e61-f3c3-4fca-b11f-c018048b96aa]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1fdeaa47-6a47-4857-bf09-f43c1c253773]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:48.851871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fdeaa47-6a47-4857-bf09-f43c1c253773]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:48.854340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:48.857843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:48.862228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc37b49a-329d-49eb-b2e4-3b2ccfa12ea2]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.863220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:48.863570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:48.863828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:01:48.864172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:48.869080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6639efaf-baff-4136-9ccb-cadfc375ea23]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.870374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5389463a-c300-4e62-8420-658f3360d4cc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.871917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acbd6ea0-ef54-4cf6-b134-625ea5070a1f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.900545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d5d42db-b858-48d4-9847-6f6ae2ff0ed1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 820479, 'reachable_time': 42692, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230661, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.909051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:01:48.909335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08dbfb0b-ec91-4292-b981-88b146b430d8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:48.910362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:46Z", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:01:48.912780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3694d684-bc77-489e-8363-e652f81c0c47. Wait time elapsed: 5.563 Jul 01 11:01:48.929742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3694d684-bc77-489e-8363-e652f81c0c47. Wait time elapsed: 5.564 Jul 01 11:01:48.929742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:52.025813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 5.879s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:52.026460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:01:52.027262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (391e582a-f7fd-476c-8bcf-9c33d2f0da0c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:01:52.027717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:52.028098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:52.029476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d8b2f92-3e2d-42e3-999b-2de75ae363b9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.058109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f28f9fd-8ce4-4dbf-a675-3c334b9feda1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.111079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.199s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:52.111316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:01:52.113615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:01:52.114203 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:52.114477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:52.115332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de5f9a6f-4ba5-4fff-a929-e36a35428ba7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.115669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2deb192-1317-4443-a475-389c07568c29]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.137992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5c9c0ac-8f99-42fb-a990-7999c0c6f2e1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.163650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf8f64aa-7c34-4596-9af2-4a6ec93c8976]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.195535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3314a7db-0a3b-4b19-8924-e0b53b33e331]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230668, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.213766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df084ef6-9d66-4e17-bd00-78da457282c0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230672, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.223611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d714c5a6-eaf1-4a98-a6f8-6a05bc7cf336]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.245826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[af272141-53e1-44ad-af7d-d70d75ead22f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.257534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d75c3c58-02b4-4043-95ac-7725b084748a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.277845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47f514f1-92e7-4fc6-802f-55376d49f30e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.288874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b09fab0-c8cc-4bb1-ab07-8c7ada84ac6b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.299440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3442fdae-8835-41c3-89a0-354c13f092eb]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.310445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0eb88671-1cd3-4bae-a75c-210bf7d05c15]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.311024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:52.311295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:52.311591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:01:52.311899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(04cbda7a-4420-4821-bdd5-1814f1d7c058) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:01:52.321437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48107e6f-30ea-43f6-b21b-fe257af1a6fb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.322241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:52.322906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:52.323238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:01:52.323669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:01:52.331408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:01:52.332104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8512fb5b-7fa7-44b4-a6de-dd818c180d7a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.337558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cceb5433-b9b2-4597-b61c-e60efba53113]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.339085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-04cbda7a-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:52.339448 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 11:01:52.341025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-04cbda7a-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:52.341025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:52.341317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:52.341703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:52.342302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-04cbda7a-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '04cbda7a-4420-4821-bdd5-1814f1d7c058', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:57:de:b2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:52.344690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:01:52.345277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e34b124d-adc3-4896-8ea0-d9904a6d8bc0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.353674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff7f1b70-3d95-433c-90ea-d0aa3d501cae]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.416388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:52.430792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2fb098bb-47cd-4158-b5b1-0f1eb906da5c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.471133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[755d8f54-c5cb-4da5-9a40-6364b301fe0d]: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:01:52.471133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:01:52.471133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:01:52.471133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:01:52.471133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:01:52.471133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:01:52.471133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:01:52.471133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:01:52.471133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:01:52.471133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:01:52.471133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:01:52.471133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:01:52.471133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:52.473574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[755d8f54-c5cb-4da5-9a40-6364b301fe0d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:52.474716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:52.478467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:52.495990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03cfb86a-32eb-4916-baed-9d87e2673fc4]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.497451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:52.498134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:52.498531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:52.498921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:52.499302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:52.545980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:52.546504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:52.546768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:01:52.547138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:52.548727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58549733-e0ec-479a-81b8-c0f30d9776a1]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.549965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d5f34127-01c4-4c80-a9cf-9b97370aa36e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.551058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35f4ad6c-eeaf-4fec-93e2-a9fab283defa]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.562171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10d15e54-9e99-43ab-8c52-d23ca4e67551]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.585524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf61e08d-6576-4a6e-8a88-ff6d62d3958c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 820845, 'reachable_time': 34388, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230705, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.595966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:01:52.596809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00099f28-66c1-431c-9203-4936397e03f5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:52.598045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1207297049-router", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3" Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:44Z", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:47Z", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b9:a0:d9", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7129, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:45Z", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:50Z", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "04cbda7a-4420-4821-bdd5-1814f1d7c058", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "893ed888-c195-4963-8385-7fb7f334adce", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:57:de:b2", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7131, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "unbound", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": {}, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:47Z", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:47Z", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 2, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:01:52.601954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:01:52.614402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 1, update_id d36eaadb-e6e6-45fc-8d20-e0b4aac74961. Wait time elapsed: 3.896 Jul 01 11:01:52.614402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 1, update_id d36eaadb-e6e6-45fc-8d20-e0b4aac74961. Wait time elapsed: 3.896 Jul 01 11:01:52.614402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:52.615215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c080c7b4-9d48-4416-8bc8-ae8fc4eb4bb5]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:01:52.615215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:01:52.615215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:01:52.615215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:01:52.615215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:01:52.615215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:01:52.615215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:01:52.615215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:01:52.615215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:01:52.615215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:01:52.615215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:01:52.615215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:01:52.615215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:52.621681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c080c7b4-9d48-4416-8bc8-ae8fc4eb4bb5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:52.628207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:52.631908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:52.635186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b51a8f7a-4e56-4ed4-b88b-b065f8dcace1]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.636450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:52.636779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:52.638559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:01:52.638926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:52.640408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[813985fa-820a-4d75-95ce-4a5737372385]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.641459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3db1f04e-0c86-46c6-8b4a-a043d87fa01a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.643317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19281b13-f298-430a-b368-a6e47fb86452]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.673930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80a5ce2d-318d-4454-a592-53eedf6de07c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 820847, 'reachable_time': 31892, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230708, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.678031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:01:52.678720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7427532d-a7af-41ed-9cf7-9295cf5a3030]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:52.680317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:50Z", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:01:52.685185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:01:52.700126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 852755d3-d750-48f8-b06a-3c6e403e3fb3. Wait time elapsed: 11.476 Jul 01 11:01:52.700126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 852755d3-d750-48f8-b06a-3c6e403e3fb3. Wait time elapsed: 11.477 Jul 01 11:01:52.700126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:54.731929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.119s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:54.732548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:01:54.733295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (391e582a-f7fd-476c-8bcf-9c33d2f0da0c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:01:54.733676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:54.733958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:54.736145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[091dfae0-1efc-4a0b-b70f-11cea8e80239]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:54.757494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4a44334-7914-4875-9801-6922df418009]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:54.782425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c07c7ee6-ed61-40a9-aba3-083b086e9bd8]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:54.815981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91918bbe-5cda-4c8e-89be-38e0c1600724]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230714, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:54.838914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7841f632-4644-4083-9825-3cc300d286c9]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:54.864691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fe1aeda-fe24-4786-b4c7-2c1545a945fe]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:54.888282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2258bc6d-6b84-4f15-8a9d-1aff94e1119f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:54.910770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d341d3d-209e-442e-9747-abe38cf55eb3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:54.912151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:54.912662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:54.913004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:01:54.913291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(04cbda7a-4420-4821-bdd5-1814f1d7c058) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:01:54.940392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:01:54.940900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb1959cd-a327-4c0b-8202-6df293894042]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:54.944120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da594b41-e335-4758-8821-bdfe0f532b63]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:54.945739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-04cbda7a-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:54.987394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-04cbda7a-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:54.988079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:54.988478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:54.988877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:54.989270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-04cbda7a-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '04cbda7a-4420-4821-bdd5-1814f1d7c058', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:57:de:b2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:55.040309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4312f18-0b35-4116-9eff-56fb5ce49e41]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:55.099818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b62f0a97-38b4-4286-8639-6123729c9a8a]: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:01:55.099818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:01:55.099818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:01:55.099818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:01:55.099818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:01:55.099818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:01:55.099818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:01:55.099818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:01:55.099818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:01:55.099818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:01:55.099818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:01:55.099818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:01:55.099818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:55.100687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b62f0a97-38b4-4286-8639-6123729c9a8a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:55.101056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:55.102409 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:55.105011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09e306d6-eb1d-445f-9dee-7d2accee3d38]: (4, ['qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:55.105011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:55.105011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:55.105011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:01:55.105459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:55.106446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67a3fac0-92e1-4a73-8998-77784ab326e9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:55.108429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f9b4cc0-e982-41fe-8dcd-d2cc4ed54194]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:55.109372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aca609db-4afa-4cbf-961f-6145cd41ab63]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:55.130762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[301c3d96-90ef-44ad-8ec9-818b84de0bcb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 821107, 'reachable_time': 15656, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230732, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:55.133209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:01:55.133620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcbdba32-c7ac-4ff0-9602-162442b6cd39]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:55.134749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1207297049-router", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3" Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:44Z", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:47Z", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b9:a0:d9", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7129, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:45Z", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:50Z", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "04cbda7a-4420-4821-bdd5-1814f1d7c058", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "893ed888-c195-4963-8385-7fb7f334adce", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:57:de:b2", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7131, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:47Z", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:54Z", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:01:55.139506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:01:55.150193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id 90e90923-6136-48fd-91e6-17f8ed3b2173. Wait time elapsed: 8.996 Jul 01 11:01:55.150193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id 90e90923-6136-48fd-91e6-17f8ed3b2173. Wait time elapsed: 8.996 Jul 01 11:01:55.150193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:56.169295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.479s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:56.170173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:01:56.171137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:01:56.171751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:56.172065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:56.173689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d163928-7000-4de6-96ea-f12e93352727]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.191827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.051s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:56.192100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:01:56.193331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (391e582a-f7fd-476c-8bcf-9c33d2f0da0c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:01:56.193862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:56.194138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:56.194821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52093fbf-8c7d-42ee-84af-bf372b546919]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.195969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a76afdfa-e7cd-4f91-8fc5-437a10cc9514]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.209546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acf348fb-4744-4d2b-8c47-bfa39319ab40]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.229799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f59104f5-dbb8-4d3c-a190-9a1f1cff0354]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.237054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9dfdc998-7e13-4cfc-8c42-1001833c942b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.271462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56d21150-f652-492d-ab0f-81ce29604bfe]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230798, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.272257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91d9167d-dc6a-4299-990d-0b759bbd3c0a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230799, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.295034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5388ca16-28f0-470b-93aa-a9be24fcf678]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.301040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14fe7bba-ad13-490e-bb6c-5aec7eb00665]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.318144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e7db50e-e52b-44ec-9581-15ca6faf703c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.323782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[174fd512-fee6-440d-9c8d-fb4aad3dfb8a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.341670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4332dec3-f51c-4050-9cc4-21b02d17718b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.349930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0bf134f0-dcd3-4cf5-913b-7816af3ff75e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.362396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0bb2408e-d62c-4a3e-8a17-da481af6e974]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.363814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:56.364332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:56.364882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:01:56.365367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:01:56.385611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[917f99a0-e6fe-40ae-b285-7504c70d8818]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.386590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:56.387078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:56.387959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:01:56.388439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(04cbda7a-4420-4821-bdd5-1814f1d7c058) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:01:56.396782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:01:56.397400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad1c82b9-37eb-4b8f-ada2-8af7414b6017]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.404794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00cf7048-7c42-4177-948e-ea887e7c344b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.408388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:56.412202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:01:56.423406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2dbb7af6-3e01-4b7c-8e72-f4c2e14ac5aa]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.428290 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d18c99e-b500-46a5-b9c7-e5fcae529754]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.444358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-04cbda7a-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:56.489658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:56.491293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:56.491877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:56.492423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:56.492967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:56.545923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-04cbda7a-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:56.550276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:56.551852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:56.553848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:56.554483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-04cbda7a-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '04cbda7a-4420-4821-bdd5-1814f1d7c058', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:57:de:b2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:56.556121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d230e03e-6bdb-4aa0-adda-8f09ea974477]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.601669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4cf023ed-c190-4b62-8db4-2dbd04de1292]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:01:56.601669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:01:56.601669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:01:56.601669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:01:56.601669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:01:56.601669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:01:56.601669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:01:56.601669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:01:56.601669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:01:56.601669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:01:56.601669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:01:56.601669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:01:56.601669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4cf023ed-c190-4b62-8db4-2dbd04de1292]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:56.607436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:56.611009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:56.613962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85fbf969-ad41-4c78-a563-d0124a334873]: (4, ['qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.614866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3920b399-3ee4-4a86-8ca4-20569980bb9f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.616021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:56.616631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:56.617070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:01:56.617607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:56.619819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b3e38db-3f67-417d-8daf-993f80a3a9a2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.621114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea9ac28f-3098-4884-8fba-3453721d481b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.622826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8919f394-0e95-4a3f-831b-1794d22abba4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.651526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[412d2e2e-aca9-41c4-9246-117ce831ac77]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 821253, 'reachable_time': 21316, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230835, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1502d7cd-cbc5-4b37-9d78-0cf932a8e912]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:01:56.656726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:53Z", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3694d684-bc77-489e-8363-e652f81c0c47. Wait time elapsed: 13.308 Jul 01 11:01:56.661913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3694d684-bc77-489e-8363-e652f81c0c47. Wait time elapsed: 13.309 Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3bc8ef3f-a0f0-416d-81b5-55454857e87e]: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bc8ef3f-a0f0-416d-81b5-55454857e87e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:56.679282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:56.684334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:56.686005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08d250cc-6a59-434f-8756-3250fbe816cf]: (4, ['qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.686005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:56.686005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:56.686005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:01:56.686005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:56.686005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[844276f8-ae4d-4811-a36c-e728bf4a763b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.686005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21a11f47-07b0-42fe-82b1-71f7e9af51e5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.686005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a81c0aa3-120b-4942-972a-a9eb3a35d6cf]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.707952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5409ea48-006e-458a-95ce-3cb87d89e166]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 821253, 'reachable_time': 35660, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230837, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b722a6a-42e7-409e-9668-37069336029e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1207297049-router", Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3" Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:01:56.713240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:44Z", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:47Z", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b9:a0:d9", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7129, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:45Z", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:50Z", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "04cbda7a-4420-4821-bdd5-1814f1d7c058", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "893ed888-c195-4963-8385-7fb7f334adce", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:57:de:b2", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7131, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:47Z", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:54Z", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id d36eaadb-e6e6-45fc-8d20-e0b4aac74961. Wait time elapsed: 7.996 Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id d36eaadb-e6e6-45fc-8d20-e0b4aac74961. Wait time elapsed: 7.996 Jul 01 11:01:56.717189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:57.493837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.781s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:57.494461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:01:57.495453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (391e582a-f7fd-476c-8bcf-9c33d2f0da0c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:01:57.495997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:57.496339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:57.497570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0040543f-48ed-44b7-80bf-f147839c8c09]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.514437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[623deb4e-5156-4f95-a828-90bc79d8f8be]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.536742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95af5322-bf4c-43dd-b641-1320e411b7fc]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.569164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5cdb6bb-4aec-4b13-8a99-0c6d858729a4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230841, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.590716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b31cceb-ced6-4712-b1a1-ee15cd49e672]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.614794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df653e24-fb51-4b5b-95dd-7fb09f3bc651]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.636276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d60d860b-6f00-43ab-989d-62407f8149da]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.658939 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c48a668c-9169-48dd-a640-fc607e9b1b02]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.659582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:57.659880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:57.660145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:01:57.660578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(04cbda7a-4420-4821-bdd5-1814f1d7c058) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:01:57.682270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:01:57.682911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[837f67b5-b619-4ae8-87f6-e2bf10182fe9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.689338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6948dee0-6edd-4c99-a1cd-6b18726e065f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.690617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-04cbda7a-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:57.728039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-04cbda7a-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:57.728822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:57.729614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:57.730134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:57.730648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-04cbda7a-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '04cbda7a-4420-4821-bdd5-1814f1d7c058', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:57:de:b2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:57.785614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[feb71e36-b582-41fd-b81a-093df8ccfbb3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.835540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[bd182149-97a4-4ff8-9559-a0c2d7946b8b]: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:01:57.835540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:01:57.835540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:01:57.835540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:01:57.835540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:01:57.835540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:01:57.835540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:01:57.835540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:01:57.835540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:01:57.835540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:01:57.835540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:01:57.835540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:01:57.835540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:57.836433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd182149-97a4-4ff8-9559-a0c2d7946b8b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:57.836748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:57.838249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:57.840094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83c0c51a-8019-4736-af97-95f437aa7215]: (4, ['qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.840094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:57.840094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:57.840094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:01:57.840405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:01:57.841357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acbb908f-24ca-4f06-8882-5b593633b73a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.842945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a793315-3b48-4f0a-8a55-7498a52cc2ca]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.845752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[046191d9-db71-4afe-9269-7b4b884e8764]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.883646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5a3b440-5199-41ae-b623-1eefccc0af25]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 821383, 'reachable_time': 16248, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230860, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.894584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:01:57.895066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b5eb0fd-f0b7-4910-8dac-7719ee82ab57]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:01:57.897228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1207297049-router", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3" Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:44Z", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:47Z", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b9:a0:d9", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7129, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:45Z", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:50Z", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "04cbda7a-4420-4821-bdd5-1814f1d7c058", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "893ed888-c195-4963-8385-7fb7f334adce", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:57:de:b2", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7131, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:47Z", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:54Z", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:01:57.901795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:01:57.921849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id 90e90923-6136-48fd-91e6-17f8ed3b2173. Wait time elapsed: 11.768 Jul 01 11:01:57.921849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id 90e90923-6136-48fd-91e6-17f8ed3b2173. Wait time elapsed: 11.769 Jul 01 11:01:57.921849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:01:59.670619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.013s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:01:59.671202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:01:59.671832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:01:59.672160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:59.672439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:59.674725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65a036d0-8bbd-4890-90ca-7805d2a4fb4e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:59.688353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0299ec1a-d609-4ac7-88c7-f3ad728fc8c0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:59.709268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3330b92-9568-482f-87c2-4e84f7c654f2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:59.740913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6395835c-78a4-4615-9338-0f6d492ae5cd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230871, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:59.763181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc6ad344-e50b-4dde-8880-2d52738e54ff]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:59.791843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2134f7e5-284e-4721-b5e7-373cdf936634]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:59.815437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc900d8b-cb69-4b08-9f00-347db02f9815]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:59.835937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c43c58c4-9337-42d9-8a22-fe59fcb23e63]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:59.836498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:01:59.836780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:01:59.837045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:01:59.837372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:01:59.853476 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:01:59.853771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5db711b7-2609-429f-90fd-8c99661ea4b0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:59.857003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[145f4ecc-5be0-4f07-a113-af230af6ffce]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:01:59.857971 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:59.898842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:59.900047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:59.900707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:59.901136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:59.901948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:01:59.963199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ce4a134-a088-4e86-ba00-61cd100185b9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.012578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[03296101-5c04-4876-88a9-f9dd2b16a106]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:00.012578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:00.012578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:00.012578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:00.012578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:00.012578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:00.012578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:00.012578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:00.012578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:00.012578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:00.012578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:00.012578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:00.012578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:00.013512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03296101-5c04-4876-88a9-f9dd2b16a106]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:00.014399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:00.016059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:00.019054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de99bea9-039d-4e2c-84f3-3f5ee71fcaf2]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.019896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:00.020218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:00.020473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:00.020811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:00.022278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9db0e3bf-ea51-46d4-ab2c-6f413588daac]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.023185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[725b0869-5fd3-4b11-90d0-a6a8c793b9c8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.024256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[801bdeaf-07d6-41a6-9292-360724f7be2e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.044317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6429e74a-e778-40ee-9854-f763ddd04927]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 821600, 'reachable_time': 38568, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230890, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.046572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:00.046824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e4e4e79-f046-473d-968a-a440a73a9630]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:00.048509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:56Z", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:00.050882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 852755d3-d750-48f8-b06a-3c6e403e3fb3. Wait time elapsed: 18.837 Jul 01 11:02:00.061928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 852755d3-d750-48f8-b06a-3c6e403e3fb3. Wait time elapsed: 18.837 Jul 01 11:02:00.061928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:00.757236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.707s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:00.757877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:00.758967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:00.759586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:00.759910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:00.761195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6f6bf8b-66ab-43b3-814f-7c8d768a69f8]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.775692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b037d20-01f9-4526-9eb8-e0f48185abb6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.798487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee3db996-7951-4bee-b037-fbc1f8808127]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.829683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a738fac8-87e7-4359-851b-a33bcdfff0f1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230893, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.850930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8744641c-fa37-433e-bfc3-c0b7563d2a97]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.872390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ddae1f2-f183-4fd2-bfcb-7dba9254b211]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.894751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e161e4c-6c87-4b8b-9203-90d50f77ebfb]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.916512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c7a7a9f-6fc3-44e4-a081-fb2380db46d9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.917166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:00.917437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:00.917703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:00.918033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:00.933345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:00.933652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[decf811d-10f6-4764-923b-6d7ab5f9c874]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.936302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a70e6791-590b-4138-8743-877ef8123ab5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:00.937311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:00.976859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:00.977704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:00.978098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:00.978490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:00.978871 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:01.034115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34fbbddd-6755-4777-92fb-ecc4ca4c6346]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.071332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[129a418b-67bd-4c0f-963b-e27c99189a56]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:01.071332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:01.071332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:01.071332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:01.071332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:01.071332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:01.071332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:01.071332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:01.071332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:01.071332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:01.071332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:01.071332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:01.071332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:01.072376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[129a418b-67bd-4c0f-963b-e27c99189a56]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:01.072705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:01.074440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:01.076797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6537e0bf-f45e-41fe-b34d-d05a5e1e9da7]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.076797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:01.076797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:01.076797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:01.076797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:01.078816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9fb5ad78-4338-43e8-b37a-a103bce2c581]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.079954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d3111d5-5d8d-4082-aeeb-a03af02b8bda]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.080949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7a9be2d-2932-4eb8-8177-b2433bcdf947]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.102558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e60f26b1-dfdc-43bf-8aa3-c25a10d15be8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 821709, 'reachable_time': 21676, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230912, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.108992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:01.109249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9416617c-21c7-4197-84fe-5127c887e4d8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:01.110432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:58Z", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:01.115015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3694d684-bc77-489e-8363-e652f81c0c47. Wait time elapsed: 17.763 Jul 01 11:02:01.131188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3694d684-bc77-489e-8363-e652f81c0c47. Wait time elapsed: 17.764 Jul 01 11:02:01.131188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:01.432440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.515s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:01.433034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:01.433907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (391e582a-f7fd-476c-8bcf-9c33d2f0da0c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:01.434205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:01.434501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:01.437661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fecf254-746f-4d4a-b583-412897cffc21]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.454338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef5ab68b-89fc-45a2-98f0-874a4e8f7ad6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.477443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51ffe105-a6e3-4062-98cc-68584e794c78]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.507517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07a2d223-b3fd-4203-a8fa-67a1147918d8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230916, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.528835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8431bee0-b99f-4458-90d5-53362f2f4bff]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.552059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9101ac3b-ec37-45a6-b42d-cb9d8b15c9be]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.573465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8e2c2ab-a9d1-4399-9532-963fa51e0519]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.595816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3569f41e-e0d3-4f6e-8e74-606106d2ed67]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.596418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:01.596700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:01.596969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:01.597254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(04cbda7a-4420-4821-bdd5-1814f1d7c058) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:01.614094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:01.614393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[242284dc-b088-40be-afb3-19df3bd3a7be]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.617173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9ec9cb6-82b1-41ef-a154-6fd4d8e4f44b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.618138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-04cbda7a-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:01.648787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-04cbda7a-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:01.649583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:01.650047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:01.650579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:01.651047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-04cbda7a-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '04cbda7a-4420-4821-bdd5-1814f1d7c058', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:57:de:b2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:01.711269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e484096a-b137-4195-bdb1-9b91648dbcfe]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.744264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d4545e96-2586-4598-b03e-c3f581067bc9]: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:01.744264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:01.744264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:01.744264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:01.744264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:01.744264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:01.744264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:01.744264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:01.744264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:01.744264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:01.744264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:01.744264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:01.744264 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:01.748150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4545e96-2586-4598-b03e-c3f581067bc9]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:01.749148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:01.751734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:01.756019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f67926fc-3235-4a6a-8507-5bd1d37b4855]: (4, ['qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.756966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:01.757444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:01.758000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:01.758093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:01.759433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f34dac6-5007-4c85-941e-e5c50a537598]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.760558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd6a25e0-235d-4127-bc00-0722d1d163d9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.761606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b387598c-136e-476f-be8c-c01059d7a01e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.782825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4faf72e9-1460-4f37-b030-9d65a809297a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 821776, 'reachable_time': 36760, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230934, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.785829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:01.786263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c633996d-e384-4791-a123-9f300bb0c028]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:01.787762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1207297049-router", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3" Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:44Z", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:47Z", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b9:a0:d9", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7129, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:45Z", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:50Z", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "04cbda7a-4420-4821-bdd5-1814f1d7c058", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "893ed888-c195-4963-8385-7fb7f334adce", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:57:de:b2", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7131, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:47Z", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:59Z", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:01.792748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:01.806712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id d36eaadb-e6e6-45fc-8d20-e0b4aac74961. Wait time elapsed: 13.088 Jul 01 11:02:01.806712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id d36eaadb-e6e6-45fc-8d20-e0b4aac74961. Wait time elapsed: 13.088 Jul 01 11:02:01.806712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:02.603109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.798s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:02.603408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:02.604356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (391e582a-f7fd-476c-8bcf-9c33d2f0da0c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:02.604689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:02.604976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:02.606141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f80f0aa-5d7f-41a7-8c47-3173172600f7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.618189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f306f5e-2639-47b1-8bcc-9c48c8b43a4b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.639621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa62647e-6c1d-4c80-b2d2-e1f2df03194c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.666908 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0459c59-f2ae-4abc-9b9f-2e7ae68f4fee]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 230937, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.688032 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e065d3bb-98e5-4f81-b66b-fc3b9c58382f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.709042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1e31d23-09eb-4a5e-981f-07214711de0a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.729132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[568ef48c-840f-40bd-8808-0a0e9acda206]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.751438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68826531-3c4b-4f7f-b8e7-b26d0207f30d]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.752104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:02.752462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:02.752734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:02.753012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(04cbda7a-4420-4821-bdd5-1814f1d7c058) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:02.768600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:02.769232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c31c3861-e422-4189-a21b-b7cdcfaa9f25]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.772170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[add33985-2694-4eb0-9698-a000b42f1d2a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.773112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-04cbda7a-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:02.810752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-04cbda7a-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:02.811508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:02.811918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:02.812474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:02.813554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-04cbda7a-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '04cbda7a-4420-4821-bdd5-1814f1d7c058', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:57:de:b2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:02.862786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[401acc0b-7bf2-4b4a-9b3d-86a547936773]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.897147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d454fd4d-1105-4dba-bb52-e1b55b6053f6]: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:02.897147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:02.897147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:02.897147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:02.897147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:02.897147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:02.897147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:02.897147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:02.897147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:02.897147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:02.897147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:02.897147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:02.897147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:02.898258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d454fd4d-1105-4dba-bb52-e1b55b6053f6]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:02.899591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:02.901155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:02.903326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[258dcab1-9f2a-4223-9cb2-fa78d75ffea2]: (4, ['qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.903326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:02.903698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:02.904103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:02.904671 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:02.905960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8986586-dcb2-4c91-aac7-03dd73b1a569]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.907621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b503a51b-dd84-41ee-b6c9-578fedca6e92]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.908751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b70e1a96-7de4-41d2-b315-66b58f767613]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.929467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1854a99c-9897-4047-9fc8-1e433e3c8fec]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 821892, 'reachable_time': 27732, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 230956, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.931590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:02.932006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec35c3a2-ebd2-44ff-aad3-0b55e58aac51]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:02.935241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1207297049-router", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3" Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:44Z", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:47Z", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b9:a0:d9", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7129, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:45Z", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:50Z", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "04cbda7a-4420-4821-bdd5-1814f1d7c058", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "893ed888-c195-4963-8385-7fb7f334adce", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:57:de:b2", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7131, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:47Z", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:59Z", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:02.938823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id 90e90923-6136-48fd-91e6-17f8ed3b2173. Wait time elapsed: 16.792 Jul 01 11:02:02.948980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id 90e90923-6136-48fd-91e6-17f8ed3b2173. Wait time elapsed: 16.792 Jul 01 11:02:02.948980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:05.191120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.079s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:05.191791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:05.192415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:05.192762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:05.193037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:05.194212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b0e7278-8581-466e-8755-722242c9d81e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.208944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c83efca-1802-4c5e-a56a-ceab970ddf1c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.229494 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[338a424a-f8d6-465c-bb03-32da670bf292]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.271787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8c3213c-0406-4c54-8e03-ca57c72148e1]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231019, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.292309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fd30a42-85ae-457a-a75b-eb02c681eab7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.313355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf3fbe63-bb36-46aa-9079-c205a3b22ba0]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.334000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a235eae6-ed2c-41a5-a1f4-23644a0b47c0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.354766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98d7434c-09ae-4978-8982-21a659c4346c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.355478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:05.355700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:05.355989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:05.356267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:05.371703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:05.372004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e0dde03-3f72-4bb0-ac42-538f3df8a961]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.374696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09424697-59b5-48de-a02b-49ecbb91c40c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.376656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:05.415093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:05.424165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:05.424590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:05.424989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:05.425378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:05.490470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[155613ac-bba2-45a6-a5ab-44c0b5261dd0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.532146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0cbcbfa4-2a54-4e8f-9302-4b4ede64750e]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:05.532146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:05.532146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:05.532146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:05.532146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:05.532146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:05.532146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:05.532146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:05.532146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:05.532146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:05.532146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:05.532146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:05.532146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:05.533086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0cbcbfa4-2a54-4e8f-9302-4b4ede64750e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:05.533837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:05.535619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:05.538345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b62fc021-37ca-47c2-a5e9-1a1e3df7051a]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.538345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:05.538345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:05.538666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:05.538750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:05.539767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14cc4013-4400-407e-9356-6db1f657d545]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.540948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65e601d8-d850-46f7-b458-4d7b5af4a4d3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.542037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51219026-3a05-4e19-ab04-1030600a8658]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.576744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0358b2a-8f88-431d-865a-03877aa266ab]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 822153, 'reachable_time': 18236, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231038, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.584851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:05.585172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfd7dcdd-ff54-4a93-a5a8-aefa0188e200]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:05.586232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:02Z", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:05.588618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 852755d3-d750-48f8-b06a-3c6e403e3fb3. Wait time elapsed: 24.375 Jul 01 11:02:05.608960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 852755d3-d750-48f8-b06a-3c6e403e3fb3. Wait time elapsed: 24.375 Jul 01 11:02:05.608960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:06.304886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.717s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:06.305500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:06.306441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:06.306862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:06.307245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:06.308571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc992c8e-6aa0-4a3e-a182-cad397c29695]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.323784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d556d855-3592-42a9-bc76-119c545b265f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.345487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69d82914-d6d4-446d-86ae-d91c0ad26401]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.377385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3ca6f71-910f-47e4-a43a-9a8b6fa42fb6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231041, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.398299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1124de95-dc96-43e6-b3ae-2e396c48bc6d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.418761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d553ca0e-b851-44b6-b3ca-dba39fae867d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.442240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5d1abba-59ff-4125-badc-50c68b59b7e8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.468401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43821a9c-a0c6-4d85-b9dd-190f7db1bb61]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.468401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:06.468401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:06.468401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:06.468401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:06.486950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:06.487240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5c73b3d-ef79-40f0-8dad-19b20e3b95a0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.490001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84e1351b-bd86-4221-b28e-d9b7e24aae4d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.490975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:06.535265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:06.539092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:06.539906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:06.540385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:06.541374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:06.584408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7f9c3e8-16ff-4e56-a862-2c2edb42caf8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.627726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a7d6ec5f-f753-42be-9227-54e6aaf17b69]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:06.627726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:06.627726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:06.627726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:06.627726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:06.627726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:06.627726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:06.627726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:06.627726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:06.627726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:06.627726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:06.627726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:06.627726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:06.628750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7d6ec5f-f753-42be-9227-54e6aaf17b69]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:06.629087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:06.631027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:06.634045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e178222b-f530-4ccf-9fb4-992f04bfe72c]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.634045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:06.634045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:06.634045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:06.634045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:06.635352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d7c425a-85fa-4c63-b98f-14a429e76895]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.636302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50303a77-88bd-447e-ac53-1869b65170ee]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.637252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69e9733a-1788-48c9-bd8f-25ee2eaf0e10]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.675447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56e67140-efee-4b68-b38e-59b0c394a417]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 822264, 'reachable_time': 31912, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231060, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.682270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:06.682548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81dc2f06-e9b1-4ed3-bdcf-223709d15a8e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:06.685193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:02Z", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:06.688311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3694d684-bc77-489e-8363-e652f81c0c47. Wait time elapsed: 23.338 Jul 01 11:02:06.699590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3694d684-bc77-489e-8363-e652f81c0c47. Wait time elapsed: 23.338 Jul 01 11:02:06.699590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:06.796060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.859s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:06.796701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:06.799293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (391e582a-f7fd-476c-8bcf-9c33d2f0da0c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:06.799723 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:06.800072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:06.801743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a00defe5-53d1-4273-93be-9c3d44576e4a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.819030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[076f5045-1174-4e97-a4fd-7838da19f365]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.842465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e534d35-dd80-4a39-8187-2b8195a952ee]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.873041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30700fdb-6824-4f0b-b965-1e3d995c0020]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231063, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.894703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[318aa02e-3a94-406f-abd7-da73e599ef8f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.915898 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a36919ec-c521-400f-a682-d30b43894669]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.936383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d046b76-2de9-4c65-b6b4-56d562770512]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.957608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90987d1b-f2bf-426f-b2cb-4220208fbef5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.958192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:06.958465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:06.958730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:06.959007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(04cbda7a-4420-4821-bdd5-1814f1d7c058) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:06.974749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:06.975038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a48c3233-5eb4-4158-a287-e42dbb4a99f1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.977740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95b55384-4f33-43e0-92fa-abbca0bdda50]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:06.978697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-04cbda7a-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:07.016528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-04cbda7a-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:07.017197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:07.017595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:07.017983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:07.018368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-04cbda7a-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '04cbda7a-4420-4821-bdd5-1814f1d7c058', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:57:de:b2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:07.074929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86224e98-3822-4abc-942e-5c7bdbcc7e75]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:07.112856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1cee5b8d-5e75-4c33-854e-62b037c55866]: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:07.112856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:07.112856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:07.112856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:07.112856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:07.112856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:07.112856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:07.112856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:07.112856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:07.112856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:07.112856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:07.112856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:07.112856 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:07.113756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1cee5b8d-5e75-4c33-854e-62b037c55866]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:07.114304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:07.116785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:07.118963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89745c01-e4df-4828-b4f7-ff2ec74cfef5]: (4, ['qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:07.118963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:07.118963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:07.118963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:07.118963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:07.120347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0e0040a-a14c-492f-bfee-7c2c0d5c79db]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:07.127866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[507c6ed6-7443-4be8-8116-7e79329fe157]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:07.128904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c47c8b7a-00ff-43af-960b-508197a5c052]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:07.162187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3f80401-35e7-4b08-8a18-6b5e418192fb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 822313, 'reachable_time': 37812, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231081, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:07.177994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:07.178238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aede93f5-97ca-4255-9b3d-6c6e46b7a4b7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:07.179327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1207297049-router", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3" Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:44Z", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:47Z", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b9:a0:d9", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7129, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:45Z", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:50Z", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "04cbda7a-4420-4821-bdd5-1814f1d7c058", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "893ed888-c195-4963-8385-7fb7f334adce", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:57:de:b2", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7131, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:47Z", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:04Z", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:07.182883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:07.199894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id d36eaadb-e6e6-45fc-8d20-e0b4aac74961. Wait time elapsed: 18.467 Jul 01 11:02:07.199894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id d36eaadb-e6e6-45fc-8d20-e0b4aac74961. Wait time elapsed: 18.482 Jul 01 11:02:07.199894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:08.023984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.826s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:08.024620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:08.025223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (391e582a-f7fd-476c-8bcf-9c33d2f0da0c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:08.025547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:08.025834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:08.027069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fda4d5f-995d-41b9-9f49-6ceb09cf95e0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.051018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[de553fe2-2ff7-4975-81a2-8a95f870d4f1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.072071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ededb78-72ba-47c6-9d07-2306cf0a22bf]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.102106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd0c501b-8ddb-484d-8ead-36ea45d04adc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231085, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.123717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31dc1e78-ccfc-4fe8-b4c8-68931db36fed]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.145019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[071d8496-d4da-4734-92f8-646255562e8a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.165840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d9f6fa79-9cf5-443a-aa11-563c2465bd61]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.187163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff0c2869-02e0-4f1e-be78-a810dd2d862b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.187873 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:08.188152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:08.188428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:08.188712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(04cbda7a-4420-4821-bdd5-1814f1d7c058) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:08.203623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:08.204009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[203e836f-74e2-46d5-b702-9867141adf4e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.206786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e13f939-637b-4e1a-af7f-dccd7df23cd5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.207880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-04cbda7a-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:08.253991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-04cbda7a-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:08.254663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:08.255227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:08.256050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:08.256573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-04cbda7a-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '04cbda7a-4420-4821-bdd5-1814f1d7c058', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:57:de:b2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:08.329370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6437a3f-a16c-4594-8ba3-0c89237f9901]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.367816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5fb50968-358e-46e9-8001-800fbcb5c317]: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:08.367816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:08.367816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:08.367816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:08.367816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:08.367816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:08.367816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:08.367816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:08.367816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:08.367816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:08.367816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:08.367816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:08.367816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:08.368751 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fb50968-358e-46e9-8001-800fbcb5c317]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:08.369145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:08.370736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:08.372731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[182eee98-c71b-4b67-b365-2e25766c8d8c]: (4, ['qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.372731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:08.372731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:08.372731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:08.372731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:08.374023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c3e4aeb-1bc6-44ab-9d49-3af15783c0d5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.375532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[740bdcf9-c552-4777-ad17-3566d0835f6c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.376495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85dc6ca8-e5ff-47bb-953b-e153d8498f76]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.394919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83f80e44-d8bf-4b27-88af-b55342523ccf]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 822436, 'reachable_time': 44676, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231105, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c64f58d0-cb34-4f6b-b7d2-72094b57dfa4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1207297049-router", Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3" Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:08.399587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:44Z", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:47Z", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b9:a0:d9", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7129, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:45Z", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:50Z", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "04cbda7a-4420-4821-bdd5-1814f1d7c058", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "893ed888-c195-4963-8385-7fb7f334adce", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:57:de:b2", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7131, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:47Z", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:05Z", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id 90e90923-6136-48fd-91e6-17f8ed3b2173. Wait time elapsed: 22.255 Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id 90e90923-6136-48fd-91e6-17f8ed3b2173. Wait time elapsed: 22.255 Jul 01 11:02:08.403630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:10.026087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.339s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:10.026653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:10.027896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:10.028219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:10.028500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:10.029689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c967a79b-f636-44b7-82aa-8507ed58744a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.044771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00a5cc04-292b-4d5a-9408-f22595570b35]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.067023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c63d25d-e9a5-4b02-b652-61446746520b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.094388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[90f5db20-7b2e-408b-b2a0-99c4a9ad3aee]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231108, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.115962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[055272c6-b4c5-419e-9234-1d960169fad6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.136790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e267d5d-f5ac-4f0a-ac19-ae67c510688d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.157866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8781d68-ccfe-4ac3-abae-fdf9816b98ac]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.178225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10b4cdb2-ba3a-4252-9125-64c3e785ae68]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.178761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:10.179035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:10.179302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:10.179617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:10.195079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:10.195492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[022814c4-6335-4a18-a205-6641bbbe2c54]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.198174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bc5dc96-853f-4f40-a9c9-fdd8706dc162]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.199246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:10.240081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:10.241110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:10.241763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:10.242470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:10.242944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:10.310704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48999de4-5494-43ad-b6e0-84c6908d8863]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.350530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[32332b22-042c-450b-a416-23624079f534]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:10.350530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:10.350530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:10.350530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:10.350530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:10.350530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:10.350530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:10.350530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:10.350530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:10.350530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:10.350530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:10.350530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:10.350530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:10.351451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32332b22-042c-450b-a416-23624079f534]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:10.351994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:10.353461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:10.355304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b650b969-9dbe-4d1c-b356-301cf76e4047]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.355304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:10.355304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:10.355756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:10.356391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:10.357414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1146407-a115-44c6-8623-73bd16c9363d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.358778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[542200e3-8c54-4527-ac47-3f77b3659a69]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.360148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2f7ebc2-20d0-44bf-bebd-713c9d61c817]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.383471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[add529e9-a80b-4603-9eba-effedb290d04]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 822635, 'reachable_time': 41204, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231127, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.394878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:10.395438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e4bf094-3a5c-48ff-95b7-947b9c20860f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:10.396722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:06Z", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:10.399424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 852755d3-d750-48f8-b06a-3c6e403e3fb3. Wait time elapsed: 29.185 Jul 01 11:02:10.409281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 852755d3-d750-48f8-b06a-3c6e403e3fb3. Wait time elapsed: 29.185 Jul 01 11:02:10.409281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:11.507970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.109s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:11.509311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:11.510428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:11.511513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:11.511804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:11.517445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa400c4c-3d71-467f-962e-e6dda5165f7d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.536901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[734a9a7d-5384-40b5-9e69-a2f4aadfbdce]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.561795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f7fda8b-f264-4786-96f0-26132c077ee6]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.600639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d25dd180-f3d7-41a2-92ef-ca6502f4d04a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231190, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.628601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8ea0f3c-2d6c-47d2-bf30-176e0fc17ad6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.652617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a3943fb-c8fd-4684-8c75-286b8e97825c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.674977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b61c554f-5461-47b3-a30c-2064260098da]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.697447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21de4633-9ebd-47bc-864b-1d4b00fdde69]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.698184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:11.698739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:11.699240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:11.699688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:11.717937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:11.718380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66342618-a9aa-44ba-8896-2b0aaea671ca]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.736687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efec2ee7-8649-4f55-88ac-bf9a48a41e3b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.737937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:11.775466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:11.776433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:11.777058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:11.777823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:11.778316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:11.842829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14323f37-8637-42f3-b659-9e1aa9c01ae5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.885329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[acafe03e-9e3f-4e04-8c60-f42822705e48]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:11.885329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:11.885329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:11.885329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:11.885329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:11.885329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:11.885329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:11.885329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:11.885329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:11.885329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:11.885329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:11.885329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:11.885329 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:11.886509 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acafe03e-9e3f-4e04-8c60-f42822705e48]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:11.887519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:11.888953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:11.891385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[774d8d14-ea13-4d65-a6d6-cb7642a9e270]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.891385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:11.891385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:11.891642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:11.891896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:11.893045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c89fb567-9d08-4a31-9e68-f061ec201efd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.894473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d738d31a-3f16-412d-ad61-96820c62b246]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.895673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dabb329d-b744-41bc-a456-9e1d7ea6ed2e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.921294 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92e4dfde-2e3a-4705-8798-aa70cbb3d746]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 822785, 'reachable_time': 33468, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231208, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ddcc2553-d1e7-4087-b17a-92a2018e52f5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:11.927484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:08Z", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 13, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3 Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3694d684-bc77-489e-8363-e652f81c0c47. Wait time elapsed: 28.579 Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3694d684-bc77-489e-8363-e652f81c0c47. Wait time elapsed: 28.579 Jul 01 11:02:11.931462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:12.248893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.847s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:12.251949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:12.253148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (391e582a-f7fd-476c-8bcf-9c33d2f0da0c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:12.253711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:12.254198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:12.256579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3a2c75f-c323-4970-a026-84ee64b23323]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.274030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe07cf64-3a1e-479e-bff8-d8973efdf78d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.298380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[19785956-b2b8-4e53-8770-1a79b8cf27cf]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.333980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5241dbfe-8a12-4dd3-b06f-925915601db0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231211, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.355769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d01bf239-4e74-4954-a969-c46927d41a99]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.379485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18bdcf90-761c-4134-b0df-37200e526a03]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.403721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[408bd9b6-0617-4f90-a9ff-73db952ad515]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.426713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a1ddb52-3232-432b-a01f-2d273b7b6646]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.427295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:12.429466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:12.430110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:12.430714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(04cbda7a-4420-4821-bdd5-1814f1d7c058) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:12.462924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:12.463491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30e4379b-693d-4258-bb24-fe518809d06c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.467102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2995936b-44d8-4e37-affd-7a6ef8281c22]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.468602 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-04cbda7a-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:12.517636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-04cbda7a-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:12.519323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:12.520469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:12.520954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:12.521385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-04cbda7a-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '04cbda7a-4420-4821-bdd5-1814f1d7c058', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:57:de:b2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:12.574557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5574d0b-af77-4469-b5cc-ba5fdb421bac]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.619737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[84d877a5-3a9d-4428-9bd3-056deced6e6a]: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:12.619737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:12.619737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:12.619737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:12.619737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:12.619737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:12.619737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:12.619737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:12.619737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:12.619737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:12.619737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:12.619737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:12.619737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:12.620945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84d877a5-3a9d-4428-9bd3-056deced6e6a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:12.622230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:12.623770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:12.626357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee669d2b-e9e8-484e-9a42-277d38d403f9]: (4, ['qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.626357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:12.626357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:12.626357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:12.626357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:12.628399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4bd39744-c0e7-462e-a03a-84a9c96ce2a4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.629539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5f3bbe9-7048-488f-83fc-7eaeb1a43374]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.630915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1e57db9-2a70-4f45-aa40-0a39abb1f0c7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.653682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6010793b-0812-4b03-b2fc-c2bc98edc3ed]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 822859, 'reachable_time': 17432, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231229, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.658202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:12.658767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6f2bfcb0-c483-425d-b595-d74f0b545105]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:12.660074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1207297049-router", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3" Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:44Z", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:47Z", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b9:a0:d9", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7129, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:45Z", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:50Z", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "04cbda7a-4420-4821-bdd5-1814f1d7c058", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "893ed888-c195-4963-8385-7fb7f334adce", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:57:de:b2", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7131, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:47Z", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:10Z", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:12.663963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:12.679045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id d36eaadb-e6e6-45fc-8d20-e0b4aac74961. Wait time elapsed: 23.948 Jul 01 11:02:12.679045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id d36eaadb-e6e6-45fc-8d20-e0b4aac74961. Wait time elapsed: 23.950 Jul 01 11:02:12.679045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:13.377987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8c801fbe-52b9-48d8-a8ca-600939a2baf0 tempest-TestShelveInstance-303554393 tempest-TestShelveInstance-303554393-project] Got routers updated notification :['615b890e-df8b-448d-89b4-03af0d799292'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:02:13.494646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.825s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:13.494987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:13.496051 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (391e582a-f7fd-476c-8bcf-9c33d2f0da0c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:13.496449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:13.496828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:13.498061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c1258d3-c623-496d-8554-bf2a916d5206]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.510326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0203e951-2505-4ad5-a184-47d87aaaaa39]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.539182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2829bcca-99b3-42e4-b5dd-9661ed0ab1d8]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.570556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4d56b96-1489-4990-960a-fac8418014fe]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231232, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.596730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b75dc1cf-cbf7-48ce-99c0-0c03eef2ca8d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.622361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f050f760-4756-4d18-8636-41027ee519f6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.645849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8e2405e-0ca4-49c5-925b-5ea728ed3ccc]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.668576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[403e1431-0704-4240-bee4-aa6c2f2362f5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.669721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:13.671103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:13.674131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:13.674712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(04cbda7a-4420-4821-bdd5-1814f1d7c058) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:13.698153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:13.698359 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[350e26d7-3b3f-4325-81ef-6f8ab032f442]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.702058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0f4cc37-e44b-4ebe-a49e-d0bcbea3641c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.704162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-04cbda7a-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:13.739524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-04cbda7a-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:13.740664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:13.741065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:13.741621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:13.742015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-04cbda7a-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '04cbda7a-4420-4821-bdd5-1814f1d7c058', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:57:de:b2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:13.810296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b66a14d2-e3d1-44ac-ae00-bf7531c2360d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.855495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b97bd7e3-1a04-422c-bfd3-2d1fb22b47e9]: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:13.855495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:13.855495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:13.855495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:13.855495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:13.855495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:13.855495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:13.855495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:13.855495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:13.855495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:13.855495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:13.855495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:13.855495 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:13.856850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b97bd7e3-1a04-422c-bfd3-2d1fb22b47e9]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:13.857980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:13.860932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:13.863520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[709814ed-111c-42da-9f43-7580208540cc]: (4, ['qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.863520 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:13.864212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:13.865123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:13.866865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:13.868646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89338e75-3992-414b-88f1-cea50fb392b6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.879901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b94ea289-117d-4154-a141-5ba5fe83815b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.881145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94f13d64-8493-4a99-8a52-bfce38a706ec]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.932333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8039d64-155d-48b4-8e32-28a20e9bb10d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 822983, 'reachable_time': 33504, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231251, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.934672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:13.935070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[696efa04-ec66-421a-b956-420aab737117]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:13.936394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1207297049-router", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3" Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:44Z", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:47Z", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b9:a0:d9", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7129, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:45Z", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:50Z", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "04cbda7a-4420-4821-bdd5-1814f1d7c058", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "893ed888-c195-4963-8385-7fb7f334adce", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:57:de:b2", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7131, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:47Z", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:12Z", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:13.948587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:13.974081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id 90e90923-6136-48fd-91e6-17f8ed3b2173. Wait time elapsed: 27.805 Jul 01 11:02:13.974081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id 90e90923-6136-48fd-91e6-17f8ed3b2173. Wait time elapsed: 27.805 Jul 01 11:02:13.974081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:15.169101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:15.169830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:15.170337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:15.170566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:15.370517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.442s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:15.371235 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:15.372209 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:15.372537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:15.372978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:15.373727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:02:15.374513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f46676cd-d9f9-44fb-84fb-337f161b6ef3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.389688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c9d2c36-cf05-48d4-8c56-066d10d46f23]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.411676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f76fca84-a7dc-4f10-aa4d-b92b9bac3403]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.441543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[24b5e11f-1294-47e8-96a1-92a65a4e04d5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231255, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.461978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15f0bc9e-c355-492e-b021-c0258a0ea331]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.482794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f16015ee-b1aa-4006-9bac-034d644daf31]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.503545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e86bf710-6104-4262-aabc-b957700ad027]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.524046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72951af2-530d-4f4a-8e70-95f691f938d9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.524589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:15.524863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:15.525128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:15.525408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:15.540134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:15.540433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdf460d4-7ff0-4591-9b12-6e83c3bb3b6f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.543025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c24fdf8b-e9b8-4445-b270-e95d8bb2a770]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.544027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:15.589064 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:15.589754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:15.590291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:15.590690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:15.591084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:15.649694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[485d5581-444e-4431-b898-2f6bec7de43e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.681978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4b6862c1-2f23-4f2a-b54e-c763519e1fbc]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:15.681978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:15.681978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:15.681978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:15.681978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:15.681978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:15.681978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:15.681978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:15.681978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:15.681978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:15.681978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:15.681978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:15.681978 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:15.682823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4b6862c1-2f23-4f2a-b54e-c763519e1fbc]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:15.683119 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:15.686705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:15.688917 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ef86174-2625-4b24-bb6f-fcd17d1eb475]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.689489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:15.689820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:15.690106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:15.690479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:15.691583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[648752a6-b7ec-4dc1-9288-c45b08cae080]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.693256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81188865-5c66-49e6-8826-49c7ed15c68a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.694228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c882016f-7191-434f-8919-d1b2a2ab87ec]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.713991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0fb62c7-0731-4016-b022-8fc14b4690f5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 823170, 'reachable_time': 39952, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231274, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.720466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:15.720729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbae15fe-6cd3-43f3-bdc5-51183ce18ded]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:15.721745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c6e287da-cbaf-413a-babe-157364477c4c", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.34", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "6834cf79-9262-48d8-9ccf-52459ec36c85", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7155, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cd:22:4a", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4285f386-36c6-4467-be82-b6e6a3e1dcfc", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:02:11Z", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:11Z", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:15.724153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:11Z", Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3 Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id d8a72d95-1dc2-4d5e-aa4e-5301f48bb644. Wait time elapsed: 2.344 Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id d8a72d95-1dc2-4d5e-aa4e-5301f48bb644. Wait time elapsed: 2.345 Jul 01 11:02:15.732821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:16.444923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.721s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:16.445444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:16.449227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:16.449770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:16.450170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:16.451529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92472fe0-60c9-4465-b475-b089ad00c191]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.465894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59e2430c-68fa-42ca-9713-3b975ddc88d6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.486739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[afc6c995-4f8a-40f8-bacf-80d744af9c6d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.514766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23bf3549-00a1-4f5b-a9e8-4d70d1e50b3f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231277, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.536630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[127202f9-6a59-49da-80dd-518a8b20bc57]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.558720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[420c801e-34f4-4ba4-9058-8087216f940c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.579621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d084738c-84b3-4868-952e-b2f6f0c2a4d0]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.600628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c95ec594-5782-4114-bc28-adc3e1371d7c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.601436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:16.601742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:16.602489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:16.602828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:16.618383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:16.618747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2259abab-e7d7-48e5-8a3e-b9284b551b77]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.621336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a4d1967-2fd5-483f-86d1-f9e1b743db94]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.622276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:16.673926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:16.674505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:16.674973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:16.675322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:16.675733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:16.729298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[183b2492-0b36-41a0-a5e3-7e08ab159b47]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.768677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cc92a614-afc8-4772-ad9d-de83b96cec9f]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:16.768677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:16.768677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:16.768677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:16.768677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:16.768677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:16.768677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:16.768677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:16.768677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:16.768677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:16.768677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:16.768677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:16.768677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:16.769683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc92a614-afc8-4772-ad9d-de83b96cec9f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:16.770207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:16.771976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:16.775111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4dce4dac-474c-4ee2-adba-cd558af7e742]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.776058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:16.776380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:16.776634 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:16.776973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:16.777997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e613e644-11c2-4424-ac22-ba83b34c6f02]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.778886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09bab9a8-72da-4abb-9a50-351b346a9ab4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.779891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44d7316c-0b6b-4439-a0b0-bc4ee629b2e8]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.800063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f128b775-9052-4eea-9dfd-639da8a4b44d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 823277, 'reachable_time': 38692, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231295, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91d51bb8-5f77-4b80-9aec-55c8586a2e62]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:16.805624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c6e287da-cbaf-413a-babe-157364477c4c", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.34", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "6834cf79-9262-48d8-9ccf-52459ec36c85", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7155, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cd:22:4a", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4285f386-36c6-4467-be82-b6e6a3e1dcfc", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:02:11Z", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:11Z", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:16.809149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:14Z", Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id d8a72d95-1dc2-4d5e-aa4e-5301f48bb644. Wait time elapsed: 3.429 Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id d8a72d95-1dc2-4d5e-aa4e-5301f48bb644. Wait time elapsed: 3.430 Jul 01 11:02:16.821842 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:17.199782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.250s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:17.200271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:17.201212 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (391e582a-f7fd-476c-8bcf-9c33d2f0da0c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:17.201535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:17.201817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:17.203754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[584529b5-b333-4454-9357-b2475087641f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.218353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22416f75-01ee-49b3-ac57-200c6806ef80]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.239820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2cce674-7441-4d07-8b9a-0cd019cf110d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.269945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10c3a379-0f9f-4abe-8d6c-129c2fff4b60]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231298, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.292139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b51601a-b550-4b10-9c60-37c7b952fe55]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.313958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[122f296c-0173-4e7c-bf19-660180f15315]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.335231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd953b18-a8d0-40f1-a2ad-517f7b40e629]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.356483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b75e3837-4530-416e-8f94-2a6618929bae]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.357087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:17.357364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:17.357630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:17.357910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(04cbda7a-4420-4821-bdd5-1814f1d7c058) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:17.373814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:17.374111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f44067b1-f92e-4bed-aba3-b41e3525be0d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.377679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a90f8b76-25a8-4988-881e-84421e66554e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.379154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-04cbda7a-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:17.423774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-04cbda7a-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:17.424868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:17.425273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:17.426131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:17.426131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-04cbda7a-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '04cbda7a-4420-4821-bdd5-1814f1d7c058', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:57:de:b2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:17.478083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5db291a1-a8c2-420d-86f2-eac37c80b73b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.512662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[346da91a-6a65-4155-be9e-8755b0fccad2]: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:17.512662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:17.512662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:17.512662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:17.512662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:17.512662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:17.512662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:17.512662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:17.512662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:17.512662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:17.512662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:17.512662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:17.512662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:17.513556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[346da91a-6a65-4155-be9e-8755b0fccad2]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:17.513897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:17.516028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:17.518088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be609881-17ca-4df1-be2c-43ab8685d790]: (4, ['qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.518088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:17.518088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:17.518088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:17.518088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:17.519091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57aff2b8-cf88-4623-8c3d-d3679440e492]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.520071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b29cb531-471e-40ca-af85-b9109b761dd3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.520974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1403b218-4419-49ca-b8ef-36e8084825d5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.540598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9149c806-07bc-44de-9184-7565a7c7c8a0]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 823353, 'reachable_time': 30880, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231316, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8bc71116-3654-45ad-8db2-8c3a0bd6e246]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1207297049-router", Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3" Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:17.545065 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:44Z", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:47Z", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b9:a0:d9", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7129, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:45Z", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:50Z", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "04cbda7a-4420-4821-bdd5-1814f1d7c058", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "893ed888-c195-4963-8385-7fb7f334adce", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:57:de:b2", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7131, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:47Z", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:12Z", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3 Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id d36eaadb-e6e6-45fc-8d20-e0b4aac74961. Wait time elapsed: 28.829 Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 2, update_id d36eaadb-e6e6-45fc-8d20-e0b4aac74961. Wait time elapsed: 28.829 Jul 01 11:02:17.549425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:18.319591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.774s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:18.320082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:18.321034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (391e582a-f7fd-476c-8bcf-9c33d2f0da0c), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:18.321358 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:18.321637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:18.322965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18a0c3ea-1fac-4536-864e-c69979aeae1c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.340696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83223eff-6f17-4778-89e6-8719b13422fe]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.363094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7198fbb4-4bce-44f0-818f-7551b0b5ef8b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.391863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5602b238-cec3-414d-aeb0-a91928ab7b6e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231321, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.412529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f77b5da-21ec-4fb3-830f-4f1f2276e5e9]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.439343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[332afe73-d903-4920-bc7b-fa90d61ffa3e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.461395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[187e639e-c4cf-42d0-9f6e-a0a109588a48]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.485973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1117528f-619f-48c3-94c1-01b80ddf0cb5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.486560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:18.486893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:18.487224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:18.487614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(04cbda7a-4420-4821-bdd5-1814f1d7c058) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:18.506913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:18.507201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c52f2db0-c6d1-40b1-adc3-3d73a4292fea]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.510003 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0aaefb79-72df-4549-967d-2562d8d3eb17]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.511608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-04cbda7a-44, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:18.549543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-04cbda7a-44, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:18.550218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:18.554926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:18.555422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-04cbda7a-44, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:18.555776 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-04cbda7a-44, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '04cbda7a-4420-4821-bdd5-1814f1d7c058', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:57:de:b2'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:18.628271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2ba4a939-6b7e-4f78-aa77-d7b9424ea161]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.671081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[743611dc-8eb4-4ae7-88c4-557869841767]: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:18.671081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:18.671081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:18.671081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:18.671081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:18.671081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:18.671081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:18.671081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:18.671081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:18.671081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:18.671081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:18.671081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:18.671081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:18.673600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[743611dc-8eb4-4ae7-88c4-557869841767]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:18.675001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:18.677346 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:18.680035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[02c85985-65a5-4707-83bc-afcd5e1c063f]: (4, ['qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.680959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:18.681501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:18.683695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:18.684310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/391e582a-f7fd-476c-8bcf-9c33d2f0da0c.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:18.689585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7085027b-ad14-4210-89a8-2d5cc4112d59]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.691054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f16f2155-fd2e-48c6-bac4-13c8f1701140]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.692376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[01f83184-a063-4c12-a9d2-3c0ba55eab3a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.757540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14c014c4-d5bc-4c9c-9f9e-aff11bcdc98e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 823465, 'reachable_time': 30756, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231352, 'error': None, 'target': 'qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.766449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:18.766816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60b2368c-e51b-453a-a6c1-71d7864c19aa]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 391e582a-f7fd-476c-8bcf-9c33d2f0da0c: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-04cbda7a-44 not found in namespace qrouter-391e582a-f7fd-476c-8bcf-9c33d2f0da0c. Jul 01 11:02:18.768391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1207297049-router", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3" Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:44Z", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:47Z", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "df0e4bdd-7940-4a7d-a7cc-11309e7304dc", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:b9:a0:d9", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7129, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.3", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:45Z", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:50Z", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "04cbda7a-4420-4821-bdd5-1814f1d7c058", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "893ed888-c195-4963-8385-7fb7f334adce", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:57:de:b2", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "391e582a-f7fd-476c-8bcf-9c33d2f0da0c", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7131, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:47Z", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:16Z", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "7a6543c0c78c41b8b4ba1a8822416ecf", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "e00015eb-03d9-4e72-90d1-c8c5fff7909b", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:18.771236 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3 Jul 01 11:02:20.696716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.887s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:20.697339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:20.698313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:20.698734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:20.699197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:20.700720 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50239d56-09b7-43a2-a09d-2b92ea2f2c6a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:20.719948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e8c34baf-2122-4a36-8f59-be0aeb7b43d9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:20.741434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36b905c1-3508-4a3a-8c95-be678048c34e]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:20.772447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[853ec0e9-d3b7-4ce9-9d7f-42529606776d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231405, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:20.794975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2e09cad4-fac2-44f7-a095-5bd80edd6c9d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:20.817115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48088cd3-7958-44e6-a715-cac04a6b337d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:20.837679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08f2d35d-362d-4e82-a54a-30187df85b86]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:20.859213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a5767ab-219c-458a-8872-75e025a099ec]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:20.859994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:20.860278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:20.860547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:20.860846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:20.877395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:20.877693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efbc7051-bc03-4466-8f78-bb5f7dc21196]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:20.880997 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2508ad9c-bfac-449e-b0e9-3040d8a7df96]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:20.882007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:20.918633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:20.919578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:20.919975 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:20.920365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:20.920910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:20.962492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61768de0-baf9-4c8f-8f75-d6bfb233a4e9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:20.999334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[65516c4d-ecfe-4fef-bc13-5bd0736521eb]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:20.999334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:20.999334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:20.999334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:20.999334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:20.999334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:20.999334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:20.999334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:20.999334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:20.999334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:20.999334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:20.999334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:20.999334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:21.000973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65516c4d-ecfe-4fef-bc13-5bd0736521eb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:21.002928 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:21.005283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:21.009076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db900614-b159-40a3-ba8d-01bc9bf57545]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:21.010048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:21.010489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:21.010803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:21.011310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:21.013767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a73fb45-33df-4916-b5a3-53adeeb023d9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:21.014626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69ebdbc2-c4c0-44cb-833b-a4cec5764ed2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:21.016638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31bd39bc-cfde-4d8a-9d20-2aa460449509]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:21.039561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[529d93fa-9b34-4010-91c2-11e7143de3ce]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 823703, 'reachable_time': 44592, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231424, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d38b09d5-03e2-412d-ae89-7db34abdd6fe]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:21.044081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c6e287da-cbaf-413a-babe-157364477c4c", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.34", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "6834cf79-9262-48d8-9ccf-52459ec36c85", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7155, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cd:22:4a", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4285f386-36c6-4467-be82-b6e6a3e1dcfc", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:02:11Z", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:11Z", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:17Z", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:21.049311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id d8a72d95-1dc2-4d5e-aa4e-5301f48bb644. Wait time elapsed: 7.666 Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id d8a72d95-1dc2-4d5e-aa4e-5301f48bb644. Wait time elapsed: 7.667 Jul 01 11:02:21.061380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:23.880211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.834s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:23.881400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:23.882610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:23.883031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:23.883540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:23.884761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e2927bc-a02e-40e6-86ab-d100f1a3e4f6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:23.901904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4fb4992-ed9d-4032-b6e0-343b1f33dc53]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:23.932325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66faf2e5-2d1f-4b90-a8d5-b6efaf7c558b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:23.970239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c7ecda05-d8fe-4468-be98-e1c7fb8e19f6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231428, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:23.991156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6358410-77da-44fe-b6c1-acfc4969d454]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.013547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1f03841-66a6-415d-8d72-c4752c0ee2ec]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.034897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6b1f547-c7d5-47b3-8f7d-b984f059928e]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.056133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c4e8240-2b5a-4923-8da0-5476eaec07da]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.056758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:24.057044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:24.057315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:24.057601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:24.073854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:24.074418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6200e2b7-8e82-4200-9a29-b5b49683b1f4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.077161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70e90e4b-8224-4629-968f-abbb0ad5ec49]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.078909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:24.116794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:24.118254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:24.119217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:24.119722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:24.120545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:24.170395 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d134cfa1-2092-41f3-a8e1-7084bae578b2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.210248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f53d026f-adb8-4cc7-aff7-149ac8a46d2d]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:24.210248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:24.210248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:24.210248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:24.210248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:24.210248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:24.210248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:24.210248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:24.210248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:24.210248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:24.210248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:24.210248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:24.210248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:24.211400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f53d026f-adb8-4cc7-aff7-149ac8a46d2d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:24.212578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:24.214384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:24.216825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e27f539c-efca-4e00-81ce-e2c32bab2dfe]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.216825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:24.216825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:24.217414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:24.217414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:24.218319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d71b7ec-1360-4196-a148-d8a127e37f5c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.220275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[748f5cd3-a78d-4c98-9916-aaa30f03c829]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.221229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e37f247-b481-4844-b036-afe977e14fa9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.247066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b555d26-ff40-40be-974b-4da468572203]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 824023, 'reachable_time': 44244, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231449, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b2b707b-95ad-4b93-a062-92a6e1bd800e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:24.252307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c6e287da-cbaf-413a-babe-157364477c4c", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.34", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "6834cf79-9262-48d8-9ccf-52459ec36c85", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7155, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cd:22:4a", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4285f386-36c6-4467-be82-b6e6a3e1dcfc", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:02:11Z", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:11Z", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:21Z", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:24.256801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id d8a72d95-1dc2-4d5e-aa4e-5301f48bb644. Wait time elapsed: 10.874 Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id d8a72d95-1dc2-4d5e-aa4e-5301f48bb644. Wait time elapsed: 10.874 Jul 01 11:02:24.274595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:25.042665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.789s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:25.043281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:25.043977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:25.044307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:25.044590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:25.045985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8da6fec6-4e6c-4207-a75f-7d75b922954f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.071732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[382f93bb-f00d-4aeb-b90d-4becead8ad7a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.101655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e74367d3-d808-4fdb-8f9f-a878b0830cba]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.147896 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2cddddf-c9f5-49ec-b273-ba076a33624f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231467, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.209165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[514e054f-2a16-4374-83b5-5924eab0693e]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.290036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22df782b-f130-49be-8b8e-0f0494687368]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.356325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16c34afc-7539-433e-a7a4-2a2ebe1d65d5]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.407813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6ae6d1a-d042-4147-b2f3-e03408235948]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.408486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:25.408826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:25.409155 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:25.409497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:25.430164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:25.430698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ed896b9-dd9c-465b-accc-a8c548650288]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.436315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe1f3d3d-4d3b-48b5-b516-60071cab1415]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.438670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:25.515025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:25.517005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:25.517726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:25.518408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:25.518825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:25.602927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e0449f6-32d8-4262-9545-57cceb548119]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.647800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1e3e03f9-142b-4d7b-afd4-98f9098ac3a2]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:25.647800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:25.647800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:25.647800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:25.647800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:25.647800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:25.647800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:25.647800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:25.647800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:25.647800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:25.647800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:25.647800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:25.647800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:25.650318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e3e03f9-142b-4d7b-afd4-98f9098ac3a2]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:25.651575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:25.653424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:25.656237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[123ee998-23f0-4791-b2b0-910b6e5b35d0]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.657198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:25.657198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:25.657434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:25.657781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:25.658737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92dee6e2-69b7-451f-94f5-40102911600e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.660413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cdae5ea6-ac9c-45f2-b505-f571533a5a21]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.664166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[69a9f074-b690-4b31-9b1d-5ea4d2c18247]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.694037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4aefbf7d-e86e-45d7-91dc-7aa88ca71901]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 824140, 'reachable_time': 17264, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231531, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.699762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:25.700198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a757b64-8ab4-4d5e-a3e9-c2ec2a93c83e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:25.701707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c6e287da-cbaf-413a-babe-157364477c4c", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.34", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "6834cf79-9262-48d8-9ccf-52459ec36c85", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7155, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cd:22:4a", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4285f386-36c6-4467-be82-b6e6a3e1dcfc", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:02:11Z", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:11Z", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:25.705133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:21Z", Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 18, Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id d8a72d95-1dc2-4d5e-aa4e-5301f48bb644. Wait time elapsed: 12.326 Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id d8a72d95-1dc2-4d5e-aa4e-5301f48bb644. Wait time elapsed: 12.326 Jul 01 11:02:25.716969 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:28.495343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.790s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:28.495973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:28.496654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:28.497036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:28.497379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:28.498590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37e4d02f-d9cf-47ed-9916-1409134c387f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.515070 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9447813b-1191-42de-b7bf-f5e383057f1c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.536141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b807bc1c-2d9f-43ff-99ed-a850359d2618]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.566208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ec9735e-3045-49e9-90bb-154875f3bdf5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231539, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.586691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1c70b1e-30f0-417f-bb9f-4cbebfd0717d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.607183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d2acdfe-9506-45d3-8c20-735157645104]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.627887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d873dff4-98d5-4c1a-a910-8a602de34cea]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.648370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60456814-fa0b-4578-ba81-4fac7ad593f5]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.648894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:28.649173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:28.649439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:28.649717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:28.664524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:28.664805 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54982aa9-8e7e-41b6-b7f9-fbeacb7ed622]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.670274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea63e656-d452-49ab-8cc2-017a5ce1e48b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.671216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:28.712084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:28.712763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:28.713188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:28.713576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:28.714139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:28.783029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1cc6ee5d-2df2-4d2a-bb2c-a0463f46d842]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4088be15-5899-420e-9505-80062bc7a3d0]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:28.813004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4088be15-5899-420e-9505-80062bc7a3d0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:28.814543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:28.816086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:28.818042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d01ba8cb-836a-4e8b-a054-3f54abf644ab]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.818042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:28.818042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:28.818042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:28.818042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:28.819484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1dd11fd-ebb2-4496-809c-a130aaa957bc]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.820587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[735b73bb-4b91-43a5-b9cc-c44ea2fc1eee]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.821502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d98fd381-65ce-4888-8d6c-d9d330a570dc]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.843052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ca314c6-4bd6-40d7-b2c2-c207f2d9b23f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 824482, 'reachable_time': 23080, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231558, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.845774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:28.846184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2aa55d90-475e-4fe2-8c41-6376fa09f91b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:28.847324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c6e287da-cbaf-413a-babe-157364477c4c", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.34", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "6834cf79-9262-48d8-9ccf-52459ec36c85", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7155, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cd:22:4a", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4285f386-36c6-4467-be82-b6e6a3e1dcfc", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:02:11Z", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:11Z", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:28.851043 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:26Z", Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 20, Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id d8a72d95-1dc2-4d5e-aa4e-5301f48bb644. Wait time elapsed: 15.473 Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id d8a72d95-1dc2-4d5e-aa4e-5301f48bb644. Wait time elapsed: 15.473 Jul 01 11:02:28.864316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:02:29.685075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.833s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:02:29.685690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:02:29.686403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:02:29.686754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:29.687646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:29.689289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57cc9b95-0f11-4979-be25-93c331ab94a0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:29.704139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[46ee2333-98e0-4593-99b1-b1f6f2f226d0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:29.726237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88f1a36c-c7b8-4db1-926c-5902972d26a5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:29.758798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1d915d3-982b-4f2b-b615-346caa93d9c6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 231561, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:29.779983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fc1ea08-d9f7-44a3-92a4-e67ffa9504fa]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:29.802915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4bb37d71-652e-4acd-ab4b-847c51569c87]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:29.823727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf44a90e-32a1-402d-adc8-81ba442e5292]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:29.846099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8e66047-5806-4819-b1b0-b883794df384]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:29.847301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:02:29.847753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:02:29.848182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:02:29.848893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:02:29.869201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:02:29.869488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe871fa8-952e-4ac4-bd61-ffc783b3fcdd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:29.872718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6ebc732d-5873-43a8-9381-2cdfedd0c3ee]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:29.873747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:29.913527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:29.914740 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:29.915138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:29.915553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:29.916218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:02:29.985052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b8a5a7f-a440-4c97-ab4e-6ed82108be53]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:30.022763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[54b03284-51be-4a9f-9ccd-7de396794a4e]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:02:30.022763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:02:30.022763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:02:30.022763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:02:30.022763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:02:30.022763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:02:30.022763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:02:30.022763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:02:30.022763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:02:30.022763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:02:30.022763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:02:30.022763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:02:30.022763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:30.023895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54b03284-51be-4a9f-9ccd-7de396794a4e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:30.024712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:30.026683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:30.032388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2dde631-3793-497d-aca9-d993a747f547]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:30.032388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:30.032388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:30.032388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:02:30.032388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:02:30.034005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c832a337-9709-47d0-8b47-ef2ffcc88f21]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:30.037735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e1e5f81-6ec9-48fc-bcfa-af6a43cadb09]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:30.038830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[445a5333-8369-4312-a6d7-c9d6601c8832]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:30.061390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09fd091a-3ef6-4e83-b7ad-038babed3285]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 824602, 'reachable_time': 44072, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 231580, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:30.068555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:02:30.068810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b27c5bac-366c-46c0-b93a-bfc5209ebd3f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:02:30.069859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "c6e287da-cbaf-413a-babe-157364477c4c", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.34", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "6834cf79-9262-48d8-9ccf-52459ec36c85", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.14", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7155, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:cd:22:4a", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "4285f386-36c6-4467-be82-b6e6a3e1dcfc", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:02:11Z", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:11Z", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:02:30.072927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:28Z", Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 21, Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:02:30.085904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3 Jul 01 11:02:55.376376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:03:15.169956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:03:15.170611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:03:15.170611 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:03:15.170827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:03:35.378412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:04:15.170411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:04:15.171565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:04:15.171565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:04:15.171904 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:04:15.380229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:04:55.380803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:05:15.170545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:15.171288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:15.171288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:15.171529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:35.383206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:05:45.094891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-467ebce8-43ec-4fd0-8142-89c705df325b tempest-TestShelveInstance-303554393 tempest-TestShelveInstance-303554393-project] Got routers updated notification :['615b890e-df8b-448d-89b4-03af0d799292'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:05:45.096626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id bef1c80c-635a-4f5a-826c-73eebb6d6872. Wait time elapsed: 0.001 Jul 01 11:05:45.097132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id bef1c80c-635a-4f5a-826c-73eebb6d6872. Wait time elapsed: 0.001 Jul 01 11:05:45.097489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:05:47.748885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.651s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:05:47.749491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:05:47.750073 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:05:47.750396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:47.750676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:47.752098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47965eb1-c4f4-4a45-a9f8-24617eae2ea2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:47.767161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31f76683-48ea-4d63-8e16-142c931e27a9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:47.785009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5bdbfdc-bf2b-43c3-89c2-dffad0908d16]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:47.812271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1b89f8e-8c7f-4811-bf73-76abaab1ed7d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 234843, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:47.833760 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4625bcf-b47f-42f8-9b76-11cde0396375]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:47.855739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[92dd3c03-e914-4eea-b0e1-3349d8319b11]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:47.883965 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55f9522f-ea44-4434-ab65-814d0da638df]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:47.906555 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aa6b5eee-8a07-4bb8-8f67-353b67665688]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:47.906886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:47.907166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:47.907481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:05:47.907763 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:05:47.929546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:05:47.930559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0bcda520-23f5-48d8-a65d-7d7590332c35]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:47.937398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e72af5a-8699-49e4-aaa3-f28d4f6a0bb1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:47.938604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:47.973204 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:47.974713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:47.974713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:47.974963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:47.975630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:48.031394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ee5b75b-2c26-4260-bb9e-d4251cee1c69]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.068010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0b331c50-bbd2-4a03-a43d-f31dd9e3fe6d]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:05:48.068010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:05:48.068010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:05:48.068010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:05:48.068010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:05:48.068010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:05:48.068010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:05:48.068010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:05:48.068010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:05:48.068010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:05:48.068010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:05:48.068010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:05:48.068010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:48.069231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b331c50-bbd2-4a03-a43d-f31dd9e3fe6d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:48.070371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:48.072370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:05:48.075674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc09897b-89fc-428f-a638-b7e650aeb1d0]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.075674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:48.075674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:48.075674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:05:48.076296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:48.076947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21e4b2d7-8634-46e2-acbe-e5eab03d77aa]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.078090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[022ad395-ef68-4a2a-94c0-27a4f8a40fc2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.079958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[625a6f1e-fdc0-439d-99a9-0d0e322b6544]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.099623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[406445d0-70ce-42fe-bfcf-1cd61b636cb1]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 844407, 'reachable_time': 42040, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 234862, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.101966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:05:48.102403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[594c2cfb-b2fa-460b-a0ec-6a277ccf2158]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:48.106217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:33Z", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 25, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:05:48.109665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id bef1c80c-635a-4f5a-826c-73eebb6d6872. Wait time elapsed: 3.012 Jul 01 11:05:48.131504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id bef1c80c-635a-4f5a-826c-73eebb6d6872. Wait time elapsed: 3.012 Jul 01 11:05:48.131504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:05:48.810625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.702s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:05:48.811727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:05:48.813354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:05:48.813839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:48.815105 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:48.816201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c61e27b-30e3-4a72-8899-08b3d2c61e61]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.836132 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0774f56-4514-49b8-923c-a5b7a5ce493a]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.857202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cfb19de-c463-4be0-8282-406d1e72182c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.886360 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d5a8808-a506-4efd-8ea8-39c1a747238b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 234867, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.907210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0400483d-5e72-4cb4-a473-371e5acb4ecb]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.928024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43b52bcb-d9ed-4ef9-b72c-377a913facc9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.950138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b12a855-222d-44c9-b717-4d413a6770cf]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.970803 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a27fe36a-a1ad-4418-9270-a5566a2de69f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.971415 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:48.971696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:48.971962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:05:48.972260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:05:48.986937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:05:48.987310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63b9870c-bb7e-4924-aa25-b1cd24dfcf47]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.990063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b1fb01f-659c-4526-8826-bd9e9461855b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:48.991074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:49.024832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:49.025609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:49.027157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:49.027881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:49.028277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:49.083417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bff1d0e-22a1-458f-80fb-97da5ed5a2a2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:49.128384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8cea02ac-fa49-4c19-b178-cafeb115ff14]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:05:49.128384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:05:49.128384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:05:49.128384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:05:49.128384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:05:49.128384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:05:49.128384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:05:49.128384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:05:49.128384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:05:49.128384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:05:49.128384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:05:49.128384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:05:49.128384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:49.129722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8cea02ac-fa49-4c19-b178-cafeb115ff14]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:49.131252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:49.134354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:05:49.139354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49684330-496a-4347-9e7c-431c16565a43]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:49.140321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:49.140741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:49.141060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:05:49.141646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:49.142947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a337429b-d1da-488d-92bd-c5cb76142731]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:49.144511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f3029ba0-f0a6-404f-91f6-0760c1396b35]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:49.145591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e29ad62d-1a47-4bd2-8853-26d23772a5a6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:49.170197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eee3cd51-8395-4781-87ee-0ba123c6129f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 844514, 'reachable_time': 21124, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 234886, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:49.182944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:05:49.183350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[644bc6c6-6bbf-4d26-9309-3e14a86a0d39]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:49.184707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:02:33Z", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 25, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:05:49.188072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id bef1c80c-635a-4f5a-826c-73eebb6d6872. Wait time elapsed: 4.091 Jul 01 11:05:49.211937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id bef1c80c-635a-4f5a-826c-73eebb6d6872. Wait time elapsed: 4.092 Jul 01 11:05:49.211937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:05:52.614435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.426s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:05:52.615087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:05:52.615960 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:05:52.616918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:52.617400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:52.619754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf58911b-092e-4aa3-8031-fd2b8e89c0f9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.643451 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc2ebf24-173e-4418-a6d3-98abb9c164c9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.667464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77e1a5c9-5c33-4b84-9324-0658ea8dd3b2]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.698903 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3250af6-e17d-42e4-8fc9-c5fe4a1e5afd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 234952, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.723321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18aed5b1-6945-4574-bbcd-ec999f758335]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.746823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dad5c94b-5811-42e4-819e-e5ae5db6fad1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.769959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6bd2d32d-04a3-4091-92a3-7561e4fbc474]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.794307 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e32adab-8e01-48a7-a401-4ae6d477f75c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.795471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:52.795747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:52.796206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:05:52.796689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:05:52.814993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:05:52.815279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[49c35454-4de6-4346-9d33-87305f085938]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.818868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eac6aa64-f655-4ca0-b0e4-fb67233872a2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.819984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:52.858705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:52.859707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:52.860410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:52.860813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:52.861210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:52.929430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8683741f-09b2-497d-85a2-71bac23fc6df]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.968940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[84291199-b84f-4e30-a043-24011d6f3a79]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:05:52.968940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:05:52.968940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:05:52.968940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:05:52.968940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:05:52.968940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:05:52.968940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:05:52.968940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:05:52.968940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:05:52.968940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:05:52.968940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:05:52.968940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:05:52.968940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:52.969884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84291199-b84f-4e30-a043-24011d6f3a79]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:52.970081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:52.971996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:05:52.974005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a1df45d6-0f1d-4b6c-a929-f146a04be885]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.974005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:52.974005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:52.974271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:05:52.974472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:52.975380 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac11899d-f8e8-44fa-a090-7bc2dc700303]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.976335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23463319-051e-4655-bec0-1a499743c41a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:52.977293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed3cd68b-d5f2-4ef2-adc4-4db12a5d9297]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:53.017200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[903974d9-10bb-4b08-a92a-6750bfea095e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 844896, 'reachable_time': 33900, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 234970, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:53.025474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:05:53.025474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60657cd4-ce8a-4369-ad45-63e44897bc04]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:53.025705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:05:48Z", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 26, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:05:53.028897 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id bef1c80c-635a-4f5a-826c-73eebb6d6872. Wait time elapsed: 7.932 Jul 01 11:05:53.046029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id bef1c80c-635a-4f5a-826c-73eebb6d6872. Wait time elapsed: 7.932 Jul 01 11:05:53.046029 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:05:53.956496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.928s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:05:53.957824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:05:53.959049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:05:53.959668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:53.960165 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:53.961973 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6411e6c-9543-4b22-9bd4-bcdfcd377b72]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:53.989188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00c7b186-d676-4898-a800-e87b1a7e21b1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.013469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[221395d7-8519-4696-91ed-35060bae9cd8]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.061013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c574861-0c57-48b4-9b8f-924fd51c662a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 234974, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.091530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b15adc3-6876-496d-b17e-e7c58742c4a5]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.113981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7da26d17-40d3-41c9-88ff-784cb7a454dc]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.141512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0a14c7af-ccf6-4980-8316-4b4d88374c32]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.162372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2c3e188-0976-446c-a186-1008b913500c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.163767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:54.164077 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:54.164354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:05:54.164764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:05:54.187288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:05:54.188112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7987362-0137-4dcf-9168-8c4f4b6093ff]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.190822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07ac2791-0901-49f0-b7f5-854f974a23cf]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.191813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:54.235673 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:54.236699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:54.237304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:54.237990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:54.238683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:54.294544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ccdf06c-3d42-499a-87ea-fb68d1388c57]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[3e1d1884-8eeb-4511-a149-ae3dde4ff5e1]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:05:54.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:05:54.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:05:54.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:05:54.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:05:54.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:05:54.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:05:54.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:05:54.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:05:54.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:05:54.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:05:54.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:05:54.332591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:54.334593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3e1d1884-8eeb-4511-a149-ae3dde4ff5e1]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:54.335586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:54.337386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:05:54.340544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d94c9bff-a315-4012-8ed3-3803ef575ba8]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.340544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:54.340544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:54.340544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:05:54.340544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:54.340967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7880720f-8f71-4645-9586-9a1548c1e38c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.341766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[77e19271-e472-438b-a759-f05cbd583245]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.342948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1fff43f-e953-4b89-a6fd-2dd73a2c5514]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.366270 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d292ff82-d650-40b0-b5fe-917a0efdbcda]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 845032, 'reachable_time': 35300, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 234993, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.377892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:05:54.378252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[094e4812-de23-4b94-9d3e-4b2b5bbba851]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:54.379301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:05:52Z", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 28, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:05:54.382229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id bef1c80c-635a-4f5a-826c-73eebb6d6872. Wait time elapsed: 9.285 Jul 01 11:05:54.393631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id bef1c80c-635a-4f5a-826c-73eebb6d6872. Wait time elapsed: 9.285 Jul 01 11:05:54.393631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:05:57.513701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.132s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:05:57.514977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:05:57.516259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:05:57.517038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:57.518255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:57.523784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08c89815-931a-41f1-a7f0-8ef93f385260]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:57.559156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b7425d8-f2b1-4528-877f-9dae330f27c8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:57.599098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2deb4599-c356-49a6-bb92-d0529dc2b88c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:57.657341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4c184da-3a3d-4224-a937-c07f2b4ca60b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 235009, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:57.725529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[712eee24-eff5-4955-9b3e-3ab977236567]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:57.781685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7d5dddf-5aae-47b7-8138-ea34b65ca27a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:57.818580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4904e281-a7c9-4263-acf6-ddcd3c7c2479]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:57.856321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[875d8d04-154f-46d4-9812-557e895657b6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:57.857041 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:57.857326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:57.857675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:05:57.858022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:05:57.909208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:05:57.909467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da2f6e52-16c2-41fb-a261-172e6f8451be]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:57.933607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da83eb6f-1ce0-4b55-90ad-f7429ada7e13]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:57.938286 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:58.026915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:58.027302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:58.036296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:58.036738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:58.037143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:58.136710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91d6a46b-9072-444c-8c5e-e0c8aaba829f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:58.191817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[388e3188-88ea-4099-830e-03a68b4ff74c]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:05:58.191817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:05:58.191817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:05:58.191817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:05:58.191817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:05:58.191817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:05:58.191817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:05:58.191817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:05:58.191817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:05:58.191817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:05:58.191817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:05:58.191817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:05:58.191817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:58.193412 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[388e3188-88ea-4099-830e-03a68b4ff74c]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:58.194566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:58.200711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:05:58.204948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[256c5241-7c68-4228-b166-9303783cb334]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:58.211228 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:58.211769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:58.212187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:05:58.212709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:58.216048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f81769f0-0004-4ff8-af06-526aae65b617]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:58.217726 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[348b40bf-4701-4d7f-b150-790ef5c8c7f9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:58.236011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a75a49a7-388e-44b6-81e2-813b38859abd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:58.271538 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2bb6cf77-c4f5-4d8c-917e-2689e32a35b3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 845391, 'reachable_time': 33984, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 235079, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:58.282727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:05:58.282993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[029daf0f-9eb9-4f07-ac9e-7baa02252194]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:58.284386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:05:55Z", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 31, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:05:58.287478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id bef1c80c-635a-4f5a-826c-73eebb6d6872. Wait time elapsed: 13.189 Jul 01 11:05:58.298367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id bef1c80c-635a-4f5a-826c-73eebb6d6872. Wait time elapsed: 13.190 Jul 01 11:05:58.298367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:05:59.242449 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.956s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:05:59.244007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:05:59.246752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:05:59.247571 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:59.248311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:59.249725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[660bf01f-540a-43e2-9bc7-cf31b15f5ce6]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.279821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2cb4fef5-624d-4e30-ac7d-28914b577206]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.310017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f99b35bf-e1d3-47e8-9935-57d7a8dbe62a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.365731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd987b2e-c99c-4a21-ad56-838c68e47b44]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 235087, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.391844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1424e68a-eb85-49b7-acdf-d2601694d0d2]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.418880 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1910cbe4-442f-436c-8091-171713d21d90]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.452524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ab0fef9-7aff-431b-97cd-53bca4bc24eb]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.480929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc4c429a-ba84-4468-8e55-7e2a9ecc5493]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.481741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:05:59.482178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:05:59.482708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:05:59.483200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:05:59.512090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:05:59.512774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54441267-23eb-4322-a235-89c1408ac888]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.516381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2c210036-9ff2-4f3c-a01e-c784b6dd128a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.517559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:59.559499 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:59.560323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:59.560826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:59.561325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:59.561813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:05:59.615350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6d1118b-285c-4ad2-8e78-e01a711fa99b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.671098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e3ac3402-9c06-45a8-8dcc-e8ce313a51e7]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:05:59.671098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:05:59.671098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:05:59.671098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:05:59.671098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:05:59.671098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:05:59.671098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:05:59.671098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:05:59.671098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:05:59.671098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:05:59.671098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:05:59.671098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:05:59.671098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:59.672322 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3ac3402-9c06-45a8-8dcc-e8ce313a51e7]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:59.673347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:59.675959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:05:59.679730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36e780b3-790f-4a6e-b44a-c743014ab815]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.680718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:59.681138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:59.681462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:05:59.682158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:05:59.683578 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ed4fb36-f697-41e3-b162-cf6d6b5ae9f3]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.685071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f6f947f0-561e-424b-9d96-6e163188c03f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.686120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89230c83-8e7a-4b84-921e-3819040c2f45]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.717347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[407aea41-7d56-462c-a1f5-d042c85fb287]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 845562, 'reachable_time': 20176, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 235106, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.724099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:05:59.724398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c4905f0-d978-48e7-9707-d7870263083b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:05:59.725474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:05:58Z", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 33, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:05:59.728503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3 Jul 01 11:06:15.171252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:15.172379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:15.172938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:15.173371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:15.388144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:06:36.747138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-e0d16851-4bda-46db-9ea1-ce8184bba6c0 tempest-TestShelveInstance-303554393 tempest-TestShelveInstance-303554393-project] Got routers updated notification :['615b890e-df8b-448d-89b4-03af0d799292'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:06:36.748112 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id 187b5a15-0a79-43ac-a551-6709661c827d. Wait time elapsed: 0.001 Jul 01 11:06:36.748363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id 187b5a15-0a79-43ac-a551-6709661c827d. Wait time elapsed: 0.001 Jul 01 11:06:36.748654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:06:39.995116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.245s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:06:39.996162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:06:39.997103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:06:39.997456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:39.997854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:39.998933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1ae8c84-8911-4021-bdd7-8db568f6fd10]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.017859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33952d30-adbc-4433-9652-8206fc376db9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.040182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9c90b903-4529-4c4f-ad53-8c8e9a9428be]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.082048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[432727a5-f457-42ac-9a91-4c2d1606f169]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 235856, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.108219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[811ff299-2214-4f7a-add1-722821710294]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.131211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e53dbd26-66f4-4be4-9802-1428aabca5ec]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.153534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10407e1e-743e-4bdf-a544-63340a34e04a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.178804 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4bfdf6b4-5c5c-40b6-9688-493f89c96bd3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.179946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:40.180849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:40.180849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:06:40.181223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:06:40.200734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:06:40.201016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d8a63275-c4ec-47b1-b367-b616a7b0db84]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.204827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4b588f0-3861-4d15-99db-6c81bde332a6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.210992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:40.253364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:40.254318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:40.255487 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:40.256042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:40.256591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:40.328618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d3aff35-80d0-4618-81a5-5db204eab973]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.366404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[094ffcd1-492e-43e1-9d4a-84e697ebec03]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:06:40.366404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:06:40.366404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:06:40.366404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:06:40.366404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:06:40.366404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:06:40.366404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:06:40.366404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:06:40.366404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:06:40.366404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:06:40.366404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:06:40.366404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:06:40.366404 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:40.367526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[094ffcd1-492e-43e1-9d4a-84e697ebec03]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:40.367622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:40.370030 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:06:40.372854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[917df027-51d4-4d08-b931-7151c4efd8ee]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.372854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:40.372854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:40.372854 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:06:40.373247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:40.374387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a005b6f2-543d-42a5-a4b5-31f6e7706619]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.375439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52abf13f-44c0-4b03-83f4-c649e7b5d4a5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.376644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe902dcc-2d07-4068-87da-8082a30c019a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.400136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d006966-b9a8-457a-a0e3-b7095734cc1b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 849634, 'reachable_time': 29084, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 235875, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.402684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:06:40.403039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b17fa569-55c4-40c6-b345-7474a194b4cd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:40.404178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "668b98b4-eee2-4828-a72f-569bb65aa441", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.234", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3b9933c5-e8b1-42bb-9117-9d22f34adff2", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.13", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7411, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:03:08:10", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a54da11b-4e1f-4c36-81fe-26567223366f", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:06:35Z", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:35Z", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:06:40.407026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:02Z", Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 37, Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 187b5a15-0a79-43ac-a551-6709661c827d. Wait time elapsed: 3.658 Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 187b5a15-0a79-43ac-a551-6709661c827d. Wait time elapsed: 3.658 Jul 01 11:06:40.424226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:06:41.135721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.730s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:06:41.136333 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:06:41.139127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:06:41.139483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:41.139781 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:41.140934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b914d82-03b5-4f76-b3a8-7d0f8ae4a5c3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.150829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[658e6fc0-8284-4262-a7c4-fb8b8ee76cfe]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.174304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2448e44d-feb8-446b-bebd-38e031ce968b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.203709 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0444b7e-5214-4808-8487-e969ca28dc72]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 235878, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.225238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9fd1e38d-bd4a-4793-8ce8-4b18cb5bb9c5]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.246371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1ac2850e-1fff-4c69-bf8e-cdc1bf521618]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.266694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e43a04ec-23d5-48d9-84a4-868527f2bfc9]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.287815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc3695d2-3253-41d2-942d-768b085432e6]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.288357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:41.288636 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:41.288901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:06:41.289174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:06:41.304650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:06:41.304953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[227531dc-bb11-4590-be4e-7464b16e1e51]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.309111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d97292f-2f7a-4f73-a5f9-51679db2ce2f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.310069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:41.346832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:41.347802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:41.348197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:41.348582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:41.348966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:41.393699 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a2d217b-c626-4c83-8ff0-b4c21e90d3da]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.446135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[cdf0657b-f242-4bb7-9ef1-d98cec74e621]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:06:41.446135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:06:41.446135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:06:41.446135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:06:41.446135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:06:41.446135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:06:41.446135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:06:41.446135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:06:41.446135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:06:41.446135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:06:41.446135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:06:41.446135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:06:41.446135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:41.447532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cdf0657b-f242-4bb7-9ef1-d98cec74e621]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:41.447899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:41.450296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:06:41.453405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[656f0b1e-d6dc-4782-976d-497df8bc8cfb]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.453405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:41.453405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:41.453405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:06:41.453793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:41.454744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16aa5ea4-ba40-427c-a369-9068b6fb45ed]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.456011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c28d22c1-2c1f-45c3-850b-cbaf3ee4c7f0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.457392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb6b8d0c-6eec-4d72-9fab-145a3faea2e0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.476818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4150c36-c302-46e9-8c4b-a7afa9817fde]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 849746, 'reachable_time': 34240, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 235897, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88d0bd34-7114-4dda-8ce5-48e30945abc5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:06:41.482166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "668b98b4-eee2-4828-a72f-569bb65aa441", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.234", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3b9933c5-e8b1-42bb-9117-9d22f34adff2", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.13", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7411, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:03:08:10", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a54da11b-4e1f-4c36-81fe-26567223366f", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:06:35Z", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:35Z", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:02Z", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 37, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:06:41.487882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 187b5a15-0a79-43ac-a551-6709661c827d. Wait time elapsed: 4.735 Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 187b5a15-0a79-43ac-a551-6709661c827d. Wait time elapsed: 4.735 Jul 01 11:06:41.500601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:06:44.926091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.443s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:06:44.926707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:06:44.927287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:06:44.927858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:44.928187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:44.929824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71b6ec32-9239-4285-b3c3-8d8a20370a52]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:44.948823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c96c4d99-783a-46d9-8f4f-063a729073c8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:44.980811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[683c7983-6f12-4cfd-8c4a-b38053763271]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.009114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef3ab0f5-1a80-456f-a147-d0985dab5778]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 235964, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.033585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[101089b0-251d-41be-ad7f-014f951b809d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.054918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ba7c9ae-7ea5-4989-89fe-ddbeadfda4e5]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.075706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b88fdd7c-e723-40f6-8cd2-ba61f7c68282]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.096226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a00531ff-925a-4568-929a-011380a19fc4]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.097730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:45.098067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:45.098401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:06:45.098739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:06:45.114521 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:06:45.114823 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3c10d781-3847-4cf1-aecc-0c25c915913b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.118511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b7b60f78-5432-4022-ba79-210c525822a7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.119800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:45.155955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:45.156828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:45.157456 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:45.157855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:45.158246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:45.227300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7c48001e-4424-4fa2-a535-2b7204f7f98e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.259916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8eb6986c-5ec8-4f61-bc45-5f0647ca7ba3]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:06:45.259916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:06:45.259916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:06:45.259916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:06:45.259916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:06:45.259916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:06:45.259916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:06:45.259916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:06:45.259916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:06:45.259916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:06:45.259916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:06:45.259916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:06:45.259916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:45.261074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8eb6986c-5ec8-4f61-bc45-5f0647ca7ba3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:45.261177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:45.262876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:06:45.265066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b6aba6a-e9f1-4384-b330-f7e12873d311]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.265066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:45.265066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:45.265066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:06:45.265066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:45.265796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[889c4c56-e20e-4195-8d55-a06053850db6]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.267133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0015d159-a346-4ec8-b5b5-914ddee4dcf5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.268205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d744d268-adaa-46e2-bd42-999012998eac]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.293478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f314ff39-897f-40d5-83d6-ec9b0a261287]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 850127, 'reachable_time': 44704, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 235982, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7703330d-124e-46e9-80f5-513a3ae977ce]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:06:45.298516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "668b98b4-eee2-4828-a72f-569bb65aa441", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.234", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3b9933c5-e8b1-42bb-9117-9d22f34adff2", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.13", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7411, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:03:08:10", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a54da11b-4e1f-4c36-81fe-26567223366f", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:06:35Z", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:35Z", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:41Z", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 38, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:06:45.303241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 187b5a15-0a79-43ac-a551-6709661c827d. Wait time elapsed: 8.551 Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 187b5a15-0a79-43ac-a551-6709661c827d. Wait time elapsed: 8.551 Jul 01 11:06:45.321054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:06:46.130496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.831s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:06:46.131137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:06:46.131778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:06:46.132125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:46.132408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:46.133877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[820ab719-e0f8-43be-84a4-037f63c1635b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.144866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcf25e23-6917-4b18-90a4-11bdfee6585d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.171624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[269380b9-dd67-432d-91cf-6f8f0a8c0809]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.201193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4f908ae-61fc-4246-84be-a498e939e7f5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 235985, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.222238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[139f1f73-f6fb-4e50-8879-8bc4e8b9e968]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.242924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54d5658e-d403-48b8-a230-6d4cdeb587bc]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.260658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[07f74b47-e03f-4b40-8479-e24a877f95a3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.281469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec33c1e0-3d61-496d-b1f4-1def0c2c089e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.282138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:46.282482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:46.282847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:06:46.283195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:06:46.298707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:06:46.299117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[140a7799-90e8-49a7-bdef-b27a054c2bc5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.302014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a68cce42-6ccb-4d52-b2f0-4145ea2ad63d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.302983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:46.342829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:46.343537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:46.343933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:46.344319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:46.344700 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:46.389915 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57cb88e9-6c0a-4847-a620-ca6201d3455b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.418866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5d72921f-94c8-4948-88a2-b04a73445bff]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:06:46.418866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:06:46.418866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:06:46.418866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:06:46.418866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:06:46.418866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:06:46.418866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:06:46.418866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:06:46.418866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:06:46.418866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:06:46.418866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:06:46.418866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:06:46.418866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:46.419942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d72921f-94c8-4948-88a2-b04a73445bff]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:46.420140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:46.422881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:06:46.426736 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c23339a8-3cdf-44a0-8a06-76b0465ff2ce]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.427724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:46.432199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:46.432622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:06:46.433100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:46.434149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fcaca7f4-7361-4d15-8059-8327d9482809]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.437922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2af441bd-3c43-4cc0-b212-92fc5d5b66ab]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.439643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb1c7b2b-ff1b-41cf-99e6-9f7c0dc644a3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.460131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b37a1316-6ba2-4c99-8124-08a94d469e1d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 850246, 'reachable_time': 40816, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 236004, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.465930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:06:46.466033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9169899-9238-4d2f-b760-bcd8c0384e7c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:46.467009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "668b98b4-eee2-4828-a72f-569bb65aa441", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.234", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3b9933c5-e8b1-42bb-9117-9d22f34adff2", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.13", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7411, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:03:08:10", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a54da11b-4e1f-4c36-81fe-26567223366f", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:06:35Z", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:35Z", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:06:46.469799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:44Z", Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 40, Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 187b5a15-0a79-43ac-a551-6709661c827d. Wait time elapsed: 9.721 Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 187b5a15-0a79-43ac-a551-6709661c827d. Wait time elapsed: 9.721 Jul 01 11:06:46.482259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:06:49.423619 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.955s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:06:49.424177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:06:49.425587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:06:49.425587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:49.425587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:49.427019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb8d2a54-48cf-4167-b2cf-b4dd68b8b80d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.443998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a571e51-cc8f-45ef-be89-4cc559dbb5ee]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.466158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[298b9fe1-0919-4ddb-97ed-082189f7e765]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.495773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d531d30-1cb0-4cf5-bc49-b36da2059233]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 236012, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.517091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b1df0c9-5304-42f8-95f0-7f0db1a680b8]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.538291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b935d50-0073-4531-98a5-ee5f38b25715]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.559324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[493a6440-a754-4d63-9c0b-f73b700f2b75]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.580408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6aaa6088-449c-4f0b-925f-74b47ca1defd]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.580951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:49.581226 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:49.581491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:06:49.581791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:06:49.597303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:06:49.598185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e4f5264-e414-480e-8ec0-45ea3d1feeaa]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.600886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[005a18be-15ef-4279-83a1-6733d7aaa104]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.601907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:49.637353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:49.638022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:49.638416 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:49.638802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:49.639184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:49.683840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc454ec3-0b0b-4d25-8e1e-e6dc46d66cc6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.729272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[98d3f24a-5b30-4c99-85ab-924a3f09d60f]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:06:49.729272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:06:49.729272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:06:49.729272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:06:49.729272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:06:49.729272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:06:49.729272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:06:49.729272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:06:49.729272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:06:49.729272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:06:49.729272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:06:49.729272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:06:49.729272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:49.730401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98d3f24a-5b30-4c99-85ab-924a3f09d60f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:49.730529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:49.732433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:06:49.735026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd714f9a-4db3-4c95-b610-c3384566aa5a]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.735026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:49.735385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:49.735653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:06:49.736069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:49.737198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fa1b7aa-c20d-4f00-b1bf-cf1f5b20e11c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.738138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ca5547d-5788-48b8-8759-d0bb3d90395f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.739008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94360fa1-af24-44c0-9bb0-11e397788e5c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.758711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab502e93-a2dc-4569-83aa-26f8b02a5792]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 850575, 'reachable_time': 28080, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 236030, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0dc0235-2832-477e-ad50-3893950b4938]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:49.763815 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "668b98b4-eee2-4828-a72f-569bb65aa441", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.234", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3b9933c5-e8b1-42bb-9117-9d22f34adff2", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.13", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7411, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:03:08:10", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a54da11b-4e1f-4c36-81fe-26567223366f", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:06:35Z", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:35Z", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:06:49.767425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:46Z", Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 42, Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 187b5a15-0a79-43ac-a551-6709661c827d. Wait time elapsed: 13.017 Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 187b5a15-0a79-43ac-a551-6709661c827d. Wait time elapsed: 13.017 Jul 01 11:06:49.782934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:06:50.855302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.090s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:06:50.855988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:06:50.856769 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:06:50.857159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:50.857508 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:50.858797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d947b49d-524f-4d78-b578-145c1cccd3a1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:50.875556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efee9cea-219e-44ae-8188-d81c89c6f8ce]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:50.898836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a44280d-3480-4140-8460-3b64775c1c59]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:50.927282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87316916-77f7-451b-977b-00568d2afe25]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 236093, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:50.949154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8838bb9f-75de-4170-84ad-159e858ebd40]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:50.970343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e09bc3b8-46cf-401c-a22d-4bc03a4a9c63]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:50.992640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63d79e66-49e7-465e-91ce-b57e4d22e06f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:51.012940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[239b8a15-e9ac-4e93-93e7-2a9a2b61b0f2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:51.013491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:06:51.013976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:06:51.014245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:06:51.014529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:06:51.030167 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:06:51.030460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d556f527-50ee-4c9c-bea3-37089e4a055b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:51.033164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c452f779-0b1d-41da-b326-408ca162a7e5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:51.034205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:51.070432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:51.071107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:51.071530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:51.072564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:51.073162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:06:51.136315 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6090a67d-e6b6-42fc-a1a2-2447ed8335af]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:51.172892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9aed86f8-e907-44c7-9282-f39b963e8f55]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:06:51.172892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:06:51.172892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:06:51.172892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:06:51.172892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:06:51.172892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:06:51.172892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:06:51.172892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:06:51.172892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:06:51.172892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:06:51.172892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:06:51.172892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:06:51.172892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:51.173879 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9aed86f8-e907-44c7-9282-f39b963e8f55]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:51.174183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:51.175888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:06:51.178016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8332974-bce0-4fb3-bbaa-a041b3db35ab]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:51.178016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:51.178016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:51.178300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:06:51.178433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:06:51.179587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08632aa3-8e7a-4dfb-bd66-2e00bac85f16]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:51.180601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[138d89ac-0775-4c99-a858-945cbceaa830]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:51.181788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[514f4369-606c-47e3-96bd-e7e9d264129e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:51.204532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a8686e7-94ca-475b-a9ac-d22b3c11a78e]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 850719, 'reachable_time': 19620, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 236111, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:51.209995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:06:51.209995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[580051fd-4a17-4073-9a4a-e7e2034e8a10]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:06:51.211201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "668b98b4-eee2-4828-a72f-569bb65aa441", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.234", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "3b9933c5-e8b1-42bb-9117-9d22f34adff2", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.13", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7411, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:03:08:10", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a54da11b-4e1f-4c36-81fe-26567223366f", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:06:35Z", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:35Z", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:06:51.215306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:49Z", Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 44, Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:06:51.226028 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3 Jul 01 11:06:55.390372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:07:15.172244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:07:15.173305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:07:15.173305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:07:15.173305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:07:35.392672 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:08:15.172646 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:08:15.173388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:08:15.173388 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:08:15.173574 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:08:15.394098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:08:55.395406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:09:15.173577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:09:15.174218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:09:15.174218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:09:15.174397 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:09:35.398066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:10:09.039259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-747bd781-ab66-43ce-9206-f4e43f0b8e8a tempest-TestShelveInstance-303554393 tempest-TestShelveInstance-303554393-project] Got routers updated notification :['615b890e-df8b-448d-89b4-03af0d799292'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:10:09.040348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id 3ef6fd2e-db12-4ca3-98cf-e1fd366ccbd1. Wait time elapsed: 0.001 Jul 01 11:10:09.040603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id 3ef6fd2e-db12-4ca3-98cf-e1fd366ccbd1. Wait time elapsed: 0.001 Jul 01 11:10:09.040894 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:10.962406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.921s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:10.964727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:10.969220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:10.970283 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:10.970756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:10.972261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11742fc7-2a1d-402a-ad23-10ebae726986]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:10.993980 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3805ecb-651f-4017-9542-7974383a4d3d]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.018426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[425af04b-f1cb-46fd-bcf0-b39ac6169a8b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.053614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[44443804-ecf3-4c0b-9ea0-db0e22ceeab0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 239322, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.075503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2bd0e98-2923-43a5-8c40-e0d8283a4358]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.097537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0bcb44a-fb3c-486a-9484-d6df75dd3366]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.119983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0ba9360-d6cc-4d39-867d-1d61d87bfbc1]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.142071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc062f35-a189-4fdd-842c-2c8d6ea99fde]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.143232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:11.143232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:11.143232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:11.144039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:11.163893 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:11.164419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4771c1af-0ff9-45eb-8066-29e2d7187d30]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.167478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d13011ac-6890-4ec3-9f55-92edae5c1c79]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.168537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:11.217685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:11.219170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:11.219875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:11.220706 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:11.221391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:11.286473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b97aca2f-3522-4f0f-8fc6-e1eb9586f13a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.329816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0ece4c17-8116-4612-9738-fbc08a83e93d]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:11.329816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:11.329816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:11.329816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:11.329816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:11.329816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:11.329816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:11.329816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:11.329816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:11.329816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:11.329816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:11.329816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:11.329816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:11.331603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ece4c17-8116-4612-9738-fbc08a83e93d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:11.333958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:11.337192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:11.341695 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bd9a68a-2fb9-4967-9e85-a2e922aafdf3]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.342533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:11.342862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:11.343121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:11.343506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:11.344606 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e68c1fe-9f99-401a-9e2e-5a30b4c50c17]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.345637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b5ad7f1-91bf-4c47-b371-1b2fc1d0b33d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.346765 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2dd5a332-81d9-408e-931a-0bfc1af893ae]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.372746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[315f0605-9639-4b9e-8e6a-f4f8195e061c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 870731, 'reachable_time': 44372, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239341, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.386757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:11.387323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25586714-2ee7-4dec-9f16-c18b9f2c84df]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:11.388588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:54Z", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 49, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:11.394313 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:11.412866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3ef6fd2e-db12-4ca3-98cf-e1fd366ccbd1. Wait time elapsed: 2.355 Jul 01 11:10:11.412866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3ef6fd2e-db12-4ca3-98cf-e1fd366ccbd1. Wait time elapsed: 2.356 Jul 01 11:10:11.412866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:12.265220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.869s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:12.265830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:12.266402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:12.266719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:12.267337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:12.268548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f790f05-7d94-4a44-872d-2d2a25e44023]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.284556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[759a4b54-ed69-4227-93d2-049257bedda4]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.307945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c3a6c5f8-3f9d-4e25-b9e5-012452aca7ed]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.342662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1565702c-d026-4c67-8d6f-45c9b7c5f960]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 239349, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.364721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc20aa6c-e496-46e7-ab4f-922e7a54148b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.390239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4895f6e7-ad79-41a2-a603-121dd828a1c6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.413144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4e15be2e-3b0d-438f-9890-5d7bc8413e0a]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.443522 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eefbdbc3-01cb-48fa-849a-f3617f4922a2]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.444095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:12.444373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:12.444638 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:12.444913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:12.464447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:12.464797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4793e358-07a7-401b-b428-e006935dde39]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.467548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89a522ce-cb2d-4312-b390-a053934dbea5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.468603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:12.526579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:12.527745 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:12.528343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:12.528929 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:12.529625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:12.595859 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbf9f57d-bf32-4f40-ba7e-db316cadd598]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.626691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[7e1660d6-17bf-45f9-80f8-2a5d4348f8ad]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:12.626691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:12.626691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:12.626691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:12.626691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:12.626691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:12.626691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:12.626691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:12.626691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:12.626691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:12.626691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:12.626691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:12.626691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:12.628014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e1660d6-17bf-45f9-80f8-2a5d4348f8ad]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:12.628296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:12.631198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:12.633983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3aa2740-71bf-4508-b40b-248de568a091]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.633983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:12.633983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:12.633983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:12.633983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:12.634848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0ace1efe-4e69-4ea7-aad0-34ca2e5f1f62]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.636267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[648f1b4e-dd00-4993-a145-2b8136ab5b78]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.638439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94383440-bd70-4a26-8335-f14ff32437fd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.660952 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[50d85a98-4a76-46ea-bf94-0cb1b68bf351]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 870860, 'reachable_time': 17768, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239371, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.664992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:12.665428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03f51c22-0ee4-4d12-8a33-f2ba4aef9484]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:12.667617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:06:54Z", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 49, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:12.674124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:12.689927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3ef6fd2e-db12-4ca3-98cf-e1fd366ccbd1. Wait time elapsed: 3.637 Jul 01 11:10:12.689927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3ef6fd2e-db12-4ca3-98cf-e1fd366ccbd1. Wait time elapsed: 3.638 Jul 01 11:10:12.689927 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:15.174540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:15.175233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:15.175233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:15.175565 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:15.398187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:10:16.068189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.390s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:16.068664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:16.069795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:16.071553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:16.072153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:16.073793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b0c69a00-a8b2-47b5-b83a-a2d5c428708e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.103922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27b914e2-477f-4392-b8fb-feb362bca96c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.131683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e55567d-b8c4-473b-a3da-647303aab202]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.172605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22d8e5e6-103a-4935-b3e3-97f853f5e95b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 239456, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.208230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fd6da0c-64a8-4ac8-ac49-b157770cd5b1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.230543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac719331-1943-4f2b-8a31-d24ab1202c74]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.253821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d56487d9-12d3-4d85-aa4e-54e5916b5a67]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.276486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9242955b-d02e-4241-b9f6-0bc9c901ff02]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.277469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:16.277748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:16.278117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:16.278267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:16.296314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:16.296717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a7c75372-64c2-4ba7-861a-9c8ddac8a13e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.299564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78870702-3a91-4201-95fb-b624b3c812b2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.300529 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:16.332356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:16.333711 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:16.334783 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:16.335181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:16.335615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:16.396486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a0964c14-77cb-4ad3-a5e3-bb2c0c87e4b7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.447916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ae82a44f-334a-412e-82eb-1650d753aac9]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:16.447916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:16.447916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:16.447916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:16.447916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:16.447916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:16.447916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:16.447916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:16.447916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:16.447916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:16.447916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:16.447916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:16.447916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:16.449213 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae82a44f-334a-412e-82eb-1650d753aac9]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:16.450118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:16.453080 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:16.457111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6cad757e-e5b1-4b0d-8ce8-93eb52633651]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.457111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:16.457111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:16.457486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:16.457486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:16.459161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d7d7b8f-2f2d-46e7-bd07-79c6fd0050f4]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.460187 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3493bce7-627b-49ea-a2fd-9b704284dcdd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.461484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[babfb37d-d64c-4bb4-b40a-b8ee72fc9d5b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.496865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13303550-5963-4202-b372-036842fe2a2a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 871243, 'reachable_time': 40944, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239475, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.503940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:16.504457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[863d6e0c-feb3-4ea0-85fd-ff13745b8ad5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:16.505621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:13Z", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 50, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:16.510396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:16.526504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3ef6fd2e-db12-4ca3-98cf-e1fd366ccbd1. Wait time elapsed: 7.481 Jul 01 11:10:16.526504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3ef6fd2e-db12-4ca3-98cf-e1fd366ccbd1. Wait time elapsed: 7.481 Jul 01 11:10:16.526504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:17.333110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.812s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:17.334369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:17.335554 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:17.336291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:17.336758 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:17.338185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[451b826e-6f94-457e-bb39-5afc33403392]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.358059 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b7df84f-2533-47e0-bb18-0f9226533f2c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.382909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88b39d5d-3bc5-4add-ae68-dd5c59dabb1c]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.455269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a802633-b54a-4ef6-aa40-697dcdb3511a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 239483, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.478759 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[464c1588-f25f-40b0-afa0-c8d87c20a821]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.529356 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e103414b-30f4-4a5d-b9ae-7fe361924ef6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.575949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86e51021-5750-4558-b4b9-78fdcd9e0dd2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.608587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a94852ac-f625-4289-b577-cc7e417f1e93]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.609500 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:17.609866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:17.610229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:17.610595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(9a386d80-de9a-4056-b527-4e1743e9f40a) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:17.638970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:17.639570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b682c828-c236-4f98-aff2-eabb17dc6bd5]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.643440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a77c6b9a-b089-410a-97bc-2290e4faa145]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.644664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-9a386d80-de, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:17.681186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-9a386d80-de, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:17.682224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:17.683575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:17.684222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-9a386d80-de, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:17.684848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-9a386d80-de, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '9a386d80-de9a-4056-b527-4e1743e9f40a', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3c:87:81'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:17.736097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc7c98ba-0859-41f6-8389-c8ca2d9b4c50]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.794055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5bb62c28-2fd1-4a3d-b8dc-881503fff7b6]: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:17.794055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:17.794055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:17.794055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:17.794055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:17.794055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:17.794055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:17.794055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:17.794055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:17.794055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:17.794055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:17.794055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:17.794055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:17.795573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bb62c28-2fd1-4a3d-b8dc-881503fff7b6]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:17.796345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:17.799215 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:17.803057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00686274-22e1-4c0e-b8f5-1d2fbcd7979e]: (4, ['qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.804137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:17.804364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:17.804677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:17.805146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:17.806218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0304e5e8-6a24-4ea8-b04b-f9a991f92a3e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.807640 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2ba1b79-bb85-484b-b973-4c8d52156974]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.808834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a00edf76-0b42-4a9e-8f23-cb3754d21ab5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.837387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5bf486a0-4a8b-480b-91da-e7264cecc392]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 871371, 'reachable_time': 29116, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239506, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.845779 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:17.846820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ecc14661-2610-467a-bfbb-50f8064140f6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 615b890e-df8b-448d-89b4-03af0d799292: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-9a386d80-de not found in namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292. Jul 01 11:10:17.848036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestShelveInstance-1087408471-router", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124" Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:39Z", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:42Z", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "69f464a2-43d4-4a44-a133-61738cbc844a", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:48:10:3c", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7112, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.124", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:40Z", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:01:43Z", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9a386d80-de9a-4056-b527-4e1743e9f40a", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "2415b816-8dbb-4efa-8447-4b2949868565", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3c:87:81", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "615b890e-df8b-448d-89b4-03af0d799292", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7116, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:01:42Z", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:15Z", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 51, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "677e0386bb5141f0849a71e370bc1ef7", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "d08e9e6c-87be-473d-8f61-69486bb8356e", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:17.852362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:17.872068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3ef6fd2e-db12-4ca3-98cf-e1fd366ccbd1. Wait time elapsed: 8.822 Jul 01 11:10:17.872068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 2, update_id 3ef6fd2e-db12-4ca3-98cf-e1fd366ccbd1. Wait time elapsed: 8.822 Jul 01 11:10:17.872068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:19.464956 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c57ad7e7-1f48-4823-882b-81b23dd43d49 admin admin] Got routers updated notification :['615b890e-df8b-448d-89b4-03af0d799292'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:10:20.463457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-ef8b841f-a004-4b86-8676-bebca2e34bdc admin admin] Got router deleted notification for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 11:10:20.626177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.764s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:20.628023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 615b890e-df8b-448d-89b4-03af0d799292 info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:20.629774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:20.630964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:20.631347 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:20.633090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61474d84-29e1-47be-9ee0-64465123637a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:20.650126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d06810d-ec8c-4028-a99f-7b4e7e820cff]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:20.671049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e27e0799-fda2-4173-8312-a0e24e8974a0]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:20.699386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e874dbb-0b75-4c6f-9260-d1fa72b66fb3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 239586, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:20.725888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b78a873-670e-4c4a-9a60-5b972fdff0cf]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:20.751784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3ab9dd5-08f3-4a43-a051-bb392c80b7fa]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:20.780545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8bf9164-c44a-480e-b075-1757e21b09c3]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:20.803712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b7f0221-ddc3-4f86-978e-160a30b787da]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:20.804532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:20.805417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:20.807350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:20.843996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4320ffe-a7b8-49b6-8258-141ffe02f0b6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 871696, 'reachable_time': 32240, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239594, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:20.845420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:20.846005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:20.846676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:10:20.850341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-615b890e-df8b-448d-89b4-03af0d799292" acquired by "process_external" :: waited 0.004s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 11:10:20.882232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[605159bc-f123-444c-9648-e26adf1e232f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 871696, 'reachable_time': 32240, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239595, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:20.883735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:10:20.884523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 11:10:20.917152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0464bcce-004e-4aeb-91cf-d50eb6e9605c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.175131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8c3cc480-aea9-48ca-b8ff-742f3efc834b]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.231887 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eabe00c5-e842-446b-a32d-4a654493f229]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.491902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22840f7f-7b8f-477e-83bb-58365530e3b0]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.492844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 11:10:21.493142 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:10:21.493454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:10:21.493770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-615b890e-df8b-448d-89b4-03af0d799292" released by "process_external" :: held 0.643s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 11:10:21.494199 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:10:21.494715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-615b890e-df8b-448d-89b4-03af0d799292" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 11:10:21.495355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:10:21.495938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 11:10:21.529835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3a02ee2-08b2-4a77-a110-dea047fd8199]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.563091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7cd68a7a-5148-4050-b570-2cd86e516fa8]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.579237 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d62d9444-6cc1-4de7-86c0-2b3e8c61e2d0]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.610515 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[beac5542-337e-4e8d-9297-57d3a4d178ce]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.611157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 11:10:21.611593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:10:21.611966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:10:21.612336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-615b890e-df8b-448d-89b4-03af0d799292" released by "process_address_scope" :: held 0.118s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 11:10:21.613584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (615b890e-df8b-448d-89b4-03af0d799292), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:21.614754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:10:21.615104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-615b890e-df8b-448d-89b4-03af0d799292" acquired by "apply_metadata_nat_rules" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 11:10:21.615587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:10:21.616140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 11:10:21.631855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3aa3f4f-8fa2-4476-8b5c-4e7c73a12886]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.686546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e265640-d735-4fed-9617-f2795282f2ff]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.701839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34b7ab33-d90e-4d5e-ac01-62676f2c9bca]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:10:21 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 11:10:21 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.756485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[941db078-59a2-4994-bf84-65a7c003fd0c]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.757393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 11:10:21.757393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:10:21.757663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-615b890e-df8b-448d-89b4-03af0d799292" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:10:21.757841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-615b890e-df8b-448d-89b4-03af0d799292" released by "apply_metadata_nat_rules" :: held 0.143s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 11:10:21.758344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-615b890e-df8b-448d-89b4-03af0d799292 Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.haproxy Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 615b890e-df8b-448d-89b4-03af0d799292 Jul 01 11:10:21.758788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 11:10:21.761683 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/615b890e-df8b-448d-89b4-03af0d799292.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 11:10:21.779909 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-615b890e-df8b-448d-89b4-03af0d799292[239610]: Proxy listener started. Jul 01 11:10:21.782820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 615b890e-df8b-448d-89b4-03af0d799292 add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 11:10:21.783193 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 615b890e-df8b-448d-89b4-03af0d799292, update_id 3ef6fd2e-db12-4ca3-98cf-e1fd366ccbd1. Time elapsed: 3.922 Jul 01 11:10:21.784019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id 221997fa-faa0-4565-9498-5a3964d39190. Wait time elapsed: 2.318 Jul 01 11:10:21.784019 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 615b890e-df8b-448d-89b4-03af0d799292, action 3, priority 1, update_id 221997fa-faa0-4565-9498-5a3964d39190. Wait time elapsed: 2.319 Jul 01 11:10:21.784432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:21.815233 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.031s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:21.815527 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 615b890e-df8b-448d-89b4-03af0d799292 delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 11:10:21.815852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (615b890e-df8b-448d-89b4-03af0d799292), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:21.828851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3f940c92-381b-48a8-9419-6952ee0c5362]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.830345 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ebaba808-0ff6-4c64-af21-e50449fc7413]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.831223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[734a6daa-6d38-419f-9ef1-c3723801b151]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.831682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 11:10:21.832523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5567407f-28f9-4c8f-a3da-659bcc4ab79e]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.853727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a13ab1c-82be-4a99-9190-9d59295526dd]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 871696, 'reachable_time': 32240, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239616, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.854595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:21.855116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:21.878985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d299045-c970-48cb-a4bc-0eaa2aa8af00]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 871696, 'reachable_time': 32240, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239617, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.880298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 11:10:21.881022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:21.881131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/615b890e-df8b-448d-89b4-03af0d799292.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:21.881357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:21.882098 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 615b890e-df8b-448d-89b4-03af0d799292 {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 11:10:21.883111 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f0b0ea5-3eeb-4ca5-b623-d8acbaf95008]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.905033 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[efa27fab-046a-4a11-9eaa-1d54e2b5a96a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 871696, 'reachable_time': 32240, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239618, 'error': None, 'target': 'qrouter-615b890e-df8b-448d-89b4-03af0d799292', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.911724 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-615b890e-df8b-448d-89b4-03af0d799292 deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:21.911984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f31c988-9099-45c2-bbd6-4194541673ff]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:21.912401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 615b890e-df8b-448d-89b4-03af0d799292 delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 11:10:21.912996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (615b890e-df8b-448d-89b4-03af0d799292), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:21.913408 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:21.913756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:21.914131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:21.914458 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:21.914947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 615b890e-df8b-448d-89b4-03af0d799292, update_id 221997fa-faa0-4565-9498-5a3964d39190. Time elapsed: 0.131 Jul 01 11:10:22.991931 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-6a369251-3c55-4b02-be92-251a2d84a9f5 admin admin] Got routers updated notification :['391e582a-f7fd-476c-8bcf-9c33d2f0da0c'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:10:22.992617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 1, update_id faa68f1d-f4e1-425a-973f-25ac293faef2. Wait time elapsed: 0.000 Jul 01 11:10:22.995784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 3, priority 1, update_id faa68f1d-f4e1-425a-973f-25ac293faef2. Wait time elapsed: 0.001 Jul 01 11:10:22.996317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:23.765319 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-1e16b497-140a-49f6-a1e4-448dca0dcf11 admin admin] Got router deleted notification for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 11:10:23.984045 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.988s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:23.985402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c was not found. Performing router cleanup Jul 01 11:10:23.986691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[80def198-391f-42e7-a9ad-0cd4377bb9d6]: (4, ['qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:23.987621 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:23.987911 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:23.988166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, update_id faa68f1d-f4e1-425a-973f-25ac293faef2. Time elapsed: 0.996 Jul 01 11:10:23.988434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 1, priority 1, update_id 48fbde63-e6fc-41d2-8cb9-4bb4958cbd07. Wait time elapsed: 0.223 Jul 01 11:10:23.989489 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, action 1, priority 1, update_id 48fbde63-e6fc-41d2-8cb9-4bb4958cbd07. Wait time elapsed: 0.223 Jul 01 11:10:23.990425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Info for router 391e582a-f7fd-476c-8bcf-9c33d2f0da0c was not found. Performing router cleanup Jul 01 11:10:23.991696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e601f6fc-41b0-4bfc-9b1b-f2314007400a]: (4, ['qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:23.992661 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:23.993219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:23.993219 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 391e582a-f7fd-476c-8bcf-9c33d2f0da0c, update_id 48fbde63-e6fc-41d2-8cb9-4bb4958cbd07. Time elapsed: 0.005 Jul 01 11:10:29.099600 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c2886600-d15b-438c-9b1d-f5dd79fbb540 admin admin] Got routers updated notification :['9b77481e-d5c1-4b97-bc52-fa1e3da962ba'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:10:29.100583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 1, update_id c74e4a38-0d8d-4ec1-856f-b0c33399c139. Wait time elapsed: 0.001 Jul 01 11:10:29.100892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 1, update_id c74e4a38-0d8d-4ec1-856f-b0c33399c139. Wait time elapsed: 0.001 Jul 01 11:10:29.101298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:30.612147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.511s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:30.613265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:30.615330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:30.616247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:30.616741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:30.618352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed93f642-ac56-4bd0-bc1a-f952b26911b3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:30.641403 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33d45e31-1bd2-45ea-a668-44bbec51c5f0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:30.674090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[728ea26f-621f-4f6c-b56e-f812e0016f8b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:30.714281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[887dd0af-22f0-4600-b5c5-43a7fb9eb75d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 239754, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:30.736585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[135258db-d58e-4bb5-b369-970b3ec80789]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:30.761178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a624a0e1-eab0-443d-b6c0-5329d30e5cff]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:30.787349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[658ccfb4-2b7a-4d80-89bb-a1429d272a32]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:30.810949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a342fd4-2cdd-40df-a666-1ee28633eceb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:30.811925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:30.812364 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:30.812790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:30.838523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[369e8ef3-6950-4cd7-b37a-eaa7669cbd05]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 872697, 'reachable_time': 34524, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239760, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:30.839533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:30.840194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:30.840850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:10:30.841718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 11:10:30.842020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:30.842351 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:30.842707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': '372fc1a5-70de-43b9-bc53-1d80d2092369', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:3e:92:99', 'admin_state_up': True, 'status': 'DOWN', 'device_id': '9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'device_owner': 'network:router_gateway', 'standard_attr_id': 7642, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.76', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T11:10:27Z', 'updated_at': '2022-07-01T11:10:30Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-372fc1a5-70), ns(qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 11:10:30.868882 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-372fc1a5-70 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:30.869166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ff42e53f-ae6a-4db4-b808-0d00f62af2b3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:30.873907 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c380a5be-db80-4318-a528-ba0f40d9ce0d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:30.875669 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-372fc1a5-70, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:30.875991 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 11:10:30.877001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-372fc1a5-70, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:30.877936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-372fc1a5-70, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:30.878649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-372fc1a5-70, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:30.879339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-372fc1a5-70, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:30.880372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-372fc1a5-70, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '372fc1a5-70de-43b9-bc53-1d80d2092369', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3e:92:99'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:30.938396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[630a1b19-3758-4c77-81b3-5cf81ec7be69]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:30.972357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9b222bc4-e238-4f52-9728-fab9c75a78b0]: Network interface qg-372fc1a5-70 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:30.972357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:30.972357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:30.972357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:30.972357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:30.972357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:30.972357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:30.972357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:30.972357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:30.972357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:30.972357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:30.972357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:30.972357 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-372fc1a5-70 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:30.974344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b222bc4-e238-4f52-9728-fab9c75a78b0]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-372fc1a5-70 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:30.974344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:10:30.974790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 11:10:30.998024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05f650d1-7fe9-4ad3-83f8-dd545bdf7d4d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:31.157258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-5c809e99-50bd-4a8f-bbc3-bda76e9d74c7 admin admin] Got routers updated notification :['9b77481e-d5c1-4b97-bc52-fa1e3da962ba'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:10:31.195427 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32a57e8f-9a85-46dc-89da-a66a0c7ea754]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:31.210201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f4aeb41d-3499-4da5-a317-1084b838fe1b]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:31.378649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f18d99b5-5d07-45c2-b2ee-bdf6e56b4f07]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:31.379355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 11:10:31.380393 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:10:31.380840 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:10:31.381256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" released by "process_external" :: held 0.540s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-372fc1a5-70 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-372fc1a5-70 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-372fc1a5-70 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:31.382530 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-372fc1a5-70 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-372fc1a5-70 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:31.387160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:31.392597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6fe5cc09-4373-4a07-bada-710885fdb964]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:31.394102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:31.394446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:31.394713 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:31.395663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:31.397402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da54735d-e61f-4654-9eb5-ba163ac7784a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:31.398592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fdf6ea2-7555-4a91-9e8b-80558894ec18]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:31.399685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b3a46d55-ae70-4dea-b70e-4e112ae42358]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:31.420933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2e2a435-9c9f-4e02-8fcf-0e4372cd5223]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 872697, 'reachable_time': 34524, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239776, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:31.427884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:31.428381 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[889c3f79-68d8-4867-a7fe-fa828b250f0e]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-372fc1a5-70 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-372fc1a5-70 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:31.431473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:28Z", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 11:10:31.437328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:31.445207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 1, update_id 89824402-a4f9-4468-a7f4-3d07796a3b8f. Wait time elapsed: 0.280 Jul 01 11:10:31.445207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 1, update_id 89824402-a4f9-4468-a7f4-3d07796a3b8f. Wait time elapsed: 0.281 Jul 01 11:10:31.445207 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:33.796023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8c61f4f7-b5ed-49af-b2d2-6e1db490cacb admin admin] Got routers updated notification :['a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:10:33.796616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 1, update_id d4f80eb8-f0b6-48e6-8a84-3a3a86a74948. Wait time elapsed: 0.000 Jul 01 11:10:33.797339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 1, update_id d4f80eb8-f0b6-48e6-8a84-3a3a86a74948. Wait time elapsed: 0.001 Jul 01 11:10:33.797339 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:35.423639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.626s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:35.424586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:35.427641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:35.428272 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:35.428795 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:35.430714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3e929a3-9bce-4e8e-a84e-4686370ee6fd]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.456643 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10ef7e4b-aa03-4df8-9d5f-b82365866b11]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.478742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[980a3161-3f1e-49aa-89ae-6fd343387ab4]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.537861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ae59ed6-9caf-4f53-ba24-5ce2ef9bd2f5]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 239860, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.568314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cdcae4ed-93dd-4a0a-810f-40c35ba47cb1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.591774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[756e5183-0313-4187-add1-2f40a3cc61a6]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.615445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65494809-7f44-4833-b54e-f97eef96b7f1]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.639594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[751deb6a-293d-4971-96aa-4c7cf1ec7483]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.640161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:35.640433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:35.640748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:35.664365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f3b214f-22dc-4c89-937c-301bb1df83ce]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 873180, 'reachable_time': 20340, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239868, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.665827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:35.666211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:35.667327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:10:35.667735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 11:10:35.668171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:35.668533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.get_preserve_ips" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:35.668941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] External gateway added: port({'id': 'faff069c-a717-4354-bd44-3b7821b8b0bb', 'name': '', 'network_id': '3978e771-1614-417d-ac17-f2ea7e60d8e3', 'tenant_id': '', 'mac_address': 'fa:16:3e:3b:4b:c4', 'admin_state_up': True, 'status': 'DOWN', 'device_id': 'a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'device_owner': 'network:router_gateway', 'standard_attr_id': 7659, 'fixed_ips': [{'subnet_id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'ip_address': '172.26.5.96', 'prefixlen': 24}], 'allowed_address_pairs': [], 'extra_dhcp_opts': [], 'security_groups': [], 'description': '', 'binding:vnic_type': 'normal', 'binding:profile': {}, 'binding:host_id': 'dv-neutron-848393-1-hyperv', 'binding:vif_type': 'ovs', 'binding:vif_details': {'connectivity': 'l2', 'port_filter': True, 'ovs_hybrid_plug': False, 'datapath_type': 'system', 'bridge_name': 'br-int', 'bound_drivers': {'0': 'openvswitch'}}, 'port_security_enabled': False, 'tags': [], 'created_at': '2022-07-01T11:10:32Z', 'updated_at': '2022-07-01T11:10:34Z', 'revision_number': 3, 'project_id': '', 'subnets': [{'id': '5349a4bf-485c-463a-bf8f-e3babc2dec02', 'cidr': '172.26.5.0/24', 'gateway_ip': '172.26.5.1', 'dns_nameservers': [], 'ipv6_ra_mode': None, 'subnetpool_id': None}], 'extra_subnets': [], 'address_scopes': {'4': None, '6': None}, 'mtu': 1500}), interface(qg-faff069c-a7), ns(qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f) {{(pid=106231) _external_gateway_added /opt/stack/neutron/neutron/agent/l3/router_info.py:817}} Jul 01 11:10:35.686949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qg-faff069c-a7 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:35.687545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c06e7ae0-3559-43d7-b802-eba3e271f1b4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.691540 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6133a854-e39b-442c-87a8-8f7e01f7fbae]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.693194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qg-faff069c-a7, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:35.693564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 11:10:35.694298 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qg-faff069c-a7, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:35.695158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qg-faff069c-a7, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:35.695587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qg-faff069c-a7, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:35.695970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qg-faff069c-a7, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:35.696342 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qg-faff069c-a7, col_values=(('type', 'internal'), ('external_ids', {'iface-id': 'faff069c-a717-4354-bd44-3b7821b8b0bb', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:3b:4b:c4'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:35.735258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bdd9cb37-1f81-4116-bb24-b25934a8cbff]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.776285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[f63de4b4-21c6-42a3-8795-7752222e86c4]: Network interface qg-faff069c-a7 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:35.776285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:35.776285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:35.776285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:35.776285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:35.776285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:35.776285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:35.776285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:35.776285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:35.776285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:35.776285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:35.776285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:35.776285 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-faff069c-a7 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:35.777557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f63de4b4-21c6-42a3-8795-7752222e86c4]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qg-faff069c-a7 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.777677 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:10:35.778168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 11:10:35.793444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6842cdb-50f3-4d4f-9324-0f2ea3714e3d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.871491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-2a40be7b-e287-4dc3-83eb-8d8079be2566 admin admin] Got routers updated notification :['a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:10:35.980379 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[daa7a99e-4abd-4dfb-a05b-2acce4f95b66]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:35.996485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[878893f7-66b6-40d6-8fbf-20b6537b397d]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.111211 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.672s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:36.111618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:36.112655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:36.113053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:36.113398 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:36.116092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea32214e-265a-4a4f-8b4b-05f8b725e0ec]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.133679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c46ceee-b47d-4808-8752-3ca0a463eca9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.157281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25bc429d-020f-46ed-aed3-abb1c4a6f0a5]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.186293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[89f06373-2d03-4d15-9c77-93998b132a69]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.187450 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 11:10:36.190363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:10:36.191575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:10:36.191942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" released by "process_external" :: held 0.524s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qg-faff069c-a7 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-faff069c-a7 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.process_external() Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return caller(func, *(extras + args), **kw) Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return f(*a, **k) Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_external_gateway(ex_gw_port) Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.external_gateway_added(ex_gw_port, interface_name) Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._external_gateway_added( Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(ex_gw_port['network_id'], Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-faff069c-a7 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:36.192747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-faff069c-a7 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-faff069c-a7 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:36.197739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:36.201221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b12170aa-1074-46c6-be56-44033337258f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 239888, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.203288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e03d9e9-4cdc-4e0a-bcc7-dea5d0687ea7]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.203731 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:36.204058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:36.204318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:36.204658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:36.209176 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65b3c9ec-4656-42d4-a082-5170baba7c9d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.210317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a36e7761-586c-4362-9cda-1d205b61e98a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.211482 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c0bb246e-3ff0-47cc-81ea-02b5c64ba15f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.227655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d28ea921-069e-40c8-9b3f-d9858c615e96]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.242946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b014727c-b167-4330-99fe-b9a378997f65]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 873180, 'reachable_time': 20340, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239891, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.249242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:36.249400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[43a63adb-2beb-48b7-9f27-b87072a74461]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-faff069c-a7 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1304, in process Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.process_external() Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/decorator.py", line 232, in fun Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return caller(func, *(extras + args), **kw) Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/coordination.py", line 78, in _synchronized Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return f(*a, **k) Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1038, in process_external Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_external_gateway(ex_gw_port) Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 919, in _process_external_gateway Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.external_gateway_added(ex_gw_port, interface_name) Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 869, in external_gateway_added Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._external_gateway_added( Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 819, in _external_gateway_added Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._plug_external_gateway(ex_gw_port, interface_name, ns_name) Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 754, in _plug_external_gateway Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(ex_gw_port['network_id'], Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qg-faff069c-a7 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:36.251295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-2071907835-router", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96" Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:4b:c4", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7659, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:34Z", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 3, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv" Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 1, update_id 981275e7-b76b-4a9f-9e23-d9839b944b6a. Wait time elapsed: 0.380 Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 1, update_id 981275e7-b76b-4a9f-9e23-d9839b944b6a. Wait time elapsed: 0.380 Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:36.255725 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13420020-e3b4-4367-8d75-be4eb06fe708]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.280488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ac476e8e-dee0-482a-aa70-dece6a066bee]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.310274 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e07726a-a1e0-4b07-acc3-6abde07604a9]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.310838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:36.311118 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:36.311411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:36.311694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:36.329577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:36.329860 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9de1fe95-bca2-40b1-8261-27b3c4c986d9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.333901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[05e0e6ac-6992-4f27-b838-209cbd539cef]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.335160 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:36.336127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 11:10:36.336888 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:36.337560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:36.337963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:36.338353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:36.338738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:36.393363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c835bd52-f225-4593-9cf1-7d98e4ec4513]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.429164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0250e70d-d63e-40b1-a278-16cd88c07b57]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:36.429164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:36.429164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:36.429164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:36.429164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:36.429164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:36.429164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:36.429164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:36.429164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:36.429164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:36.429164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:36.429164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:36.429164 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0250e70d-d63e-40b1-a278-16cd88c07b57]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:36.430647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:36.432623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:36.437743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db60b5f8-26a0-4ad8-94f9-d43e2824d961]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.437743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:36.437743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:36.437743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:36.437743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:36.438891 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35577a74-fa12-4f99-955b-118337599a9b]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.439910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[433ed15a-6ccf-4f82-90ba-5b88915b77de]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.440876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[55e2bebb-1115-45c2-9a1e-381e4d49dea0]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.462185 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15d68595-a596-4ca6-8ec1-d913fb387f72]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 873245, 'reachable_time': 30696, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239906, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.467015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:36.468145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b44c2ab-d557-4ee3-952b-fa0da8d58733]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:36.469197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:34Z", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:36.472694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id c74e4a38-0d8d-4ec1-856f-b0c33399c139. Wait time elapsed: 7.370 Jul 01 11:10:36.485861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id c74e4a38-0d8d-4ec1-856f-b0c33399c139. Wait time elapsed: 7.371 Jul 01 11:10:36.485861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:37.145012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.674s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:37.145281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:37.146547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:37.146547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:37.146827 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:37.148376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e77aa06b-3552-424c-a7f5-021f2c73fa42]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.164034 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29bab56e-419b-4ec3-acd7-06b48c28e496]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.185154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58afc35f-cac9-4c07-b890-13474a270087]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.218930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[198d0f20-0d49-4a9f-984d-dcd0c6dcc1e2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 239909, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.240708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8381772a-3071-4863-be98-b7cbbc5e3fb0]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.261772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[362a5cf4-09de-4485-860c-7a241a62556a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.283031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a902251-0b8f-4d3c-956e-560fcac6a5fd]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.304573 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3df4bc88-2a11-42e4-a9fa-95167925be70]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.305959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:37.306407 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:37.306850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:37.307656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:37.324652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:37.325325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8902ca4-920c-4781-aa93-19fe3971303b]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.333721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fefbd62-7d2f-46c5-b642-d703b22e6eb3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.335336 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:37.376809 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:37.377629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:37.378605 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:37.379128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:37.380047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:37.468512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bfcb7237-3b09-4ee9-a918-264634b5fb7c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.522392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ef643e7b-8e95-441f-8c3b-39b90ebb7f38]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:37.522392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:37.522392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:37.522392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:37.522392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:37.522392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:37.522392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:37.522392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:37.522392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:37.522392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:37.522392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:37.522392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:37.522392 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:37.523916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef643e7b-8e95-441f-8c3b-39b90ebb7f38]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:37.525216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:37.527177 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:37.533748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e2a7b72-1350-41a6-ad8f-7a895350d8aa]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.535230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:37.536580 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:37.537008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:37.537568 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:37.539291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2688bccd-4130-41f0-860c-4dea2cac4bf5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.541205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e7bddd24-e915-470d-ac49-5ed864520ad0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.542371 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[841727cd-9dc0-4d0b-bfa5-c0cddd28f89d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.573497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c28f399c-d645-4000-bb0c-7c3b078c055a]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 873348, 'reachable_time': 23804, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 239927, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.581245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:37.581543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7ad42648-a9ea-46d0-8a00-169538160dba]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:37.582603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:34Z", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:37.585798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 89824402-a4f9-4468-a7f4-3d07796a3b8f. Wait time elapsed: 6.426 Jul 01 11:10:37.603727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 89824402-a4f9-4468-a7f4-3d07796a3b8f. Wait time elapsed: 6.426 Jul 01 11:10:37.603727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:41.616068 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 4.032s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:41.616730 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:41.617645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:41.618063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:41.618440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:41.619708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[244108a0-de95-4228-a6cc-2f97caa8289e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:41.639384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2a23b9d-2c62-4a78-ab17-1d06d5bde3ca]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:41.662598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d29525b0-271c-4a31-9353-952aaf5d8c5b]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:41.695785 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4799a43-9085-45e7-a802-7bc415b82e3e]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240001, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:41.722222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54a406b1-09ae-403a-ae1f-b90a6a6b9311]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:41.743990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fd320fa5-e700-42b2-8e77-100eab83e844]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:41.768437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[146926fb-2a20-47c9-be21-9e5d9cf5e9ba]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:41.791243 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[db445490-6969-4451-82a5-7f8c3993f1cb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:41.791868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:41.792144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:41.792413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:41.792690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:41.809506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:41.809945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffbe417a-1b14-4aff-808d-a39dd7f20d15]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:41.813121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e000fbf3-86a4-4148-ae46-fb4d4593fcef]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:41.814146 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:41.852925 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:41.857845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:41.858474 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:41.861478 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:41.862069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:41.943848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a14610b0-acd2-4822-863b-6dc6d768a600]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:41.992424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[edada655-a3eb-4585-841f-34f7cd911f29]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:41.992424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:41.992424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:41.992424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:41.992424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:41.992424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:41.992424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:41.992424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:41.992424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:41.992424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:41.992424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:41.992424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:41.992424 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:41.994878 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[edada655-a3eb-4585-841f-34f7cd911f29]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:42.000122 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:42.003429 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:42.009934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3584bdf6-a05a-40f8-92df-61608bc5159e]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:42.011161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:42.011641 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:42.011905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:42.012362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:42.013462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4045814b-d72f-4076-bdc9-b491849c7433]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:42.015181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fd4d884-5ad7-4b4e-965a-d8b0ea6d8713]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:42.016594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cc98d563-2070-4261-9365-5229a646282a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:42.039753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88591a64-1363-4113-98b4-f6018d024d27]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 873795, 'reachable_time': 28872, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240019, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a438382-ae39-48db-8320-d170b0be1c71]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:42.044267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:34Z", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id c74e4a38-0d8d-4ec1-856f-b0c33399c139. Wait time elapsed: 12.944 Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id c74e4a38-0d8d-4ec1-856f-b0c33399c139. Wait time elapsed: 12.946 Jul 01 11:10:42.052201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:42.881793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.836s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:42.882436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:42.883698 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:42.884096 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:42.885069 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:42.886693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ebaa9593-6e0f-4301-973a-ec5924d6c680]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:42.904951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fa2e9b4-a9e7-4e2a-955c-958de1a1c9de]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:42.930942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e5a12a4b-3f27-4f7a-8bb4-ac9c37fdfd74]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:42.972486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9173349b-e06e-4a9c-9d39-5e1cef90d23a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240022, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:42.998901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63e42226-b811-4d79-9ac8-4b55b8708732]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.020994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[23596991-5f4c-474d-974d-67a6f6a123e3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.042692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d982d61d-7e42-4509-a697-694c4c91c326]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.069627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb0a26ea-e40d-46f3-abe8-8e1a9d198e8b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.070608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:43.071005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:43.071467 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:43.072152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:43.092958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:43.094046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4c761890-508c-4353-b56b-869115042e03]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.097651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b92e4f8-6966-42f8-b45a-a7822de1b75b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.099836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:43.150343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:43.154796 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:43.155541 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:43.160618 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:43.163085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:43.222366 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[67708a73-ab11-47d4-a253-5daa49230b44]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.268852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e6cc8c2d-9902-446a-bd1a-6d0a2e080f82]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:43.268852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:43.268852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:43.268852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:43.268852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:43.268852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:43.268852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:43.268852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:43.268852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:43.268852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:43.268852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:43.268852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:43.268852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:43.270692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e6cc8c2d-9902-446a-bd1a-6d0a2e080f82]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:43.270968 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:43.273084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:43.276338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d25648a8-e921-4e56-9f24-adabec089714]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.276338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:43.276338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:43.276338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:43.276338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:43.278126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a587ae72-679f-4682-a373-c669b6194803]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.279230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45d897dc-6774-4aac-aacc-bf8be466147d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.280253 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d00102f3-29fd-4503-a1a8-4abcb8b3e580]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.304230 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b5b308f-49a0-4bc1-8f93-7f5a3586804d]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 873923, 'reachable_time': 39088, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240041, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26715431-b61a-4c39-8b92-7a8b46b45192]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:43.311386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:40Z", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 89824402-a4f9-4468-a7f4-3d07796a3b8f. Wait time elapsed: 12.151 Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 89824402-a4f9-4468-a7f4-3d07796a3b8f. Wait time elapsed: 12.151 Jul 01 11:10:43.317717 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:43.568306 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 7.315s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:43.568650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:43.569655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:43.570179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:43.570468 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:43.571963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0c374fcf-3e64-4a10-adb5-1c119b8cc698]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.584244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e27c223-e51e-4540-85cf-d9f5a9ee2cc9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.605269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28437bf1-4a10-44d2-b2ce-bd8e430e208a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.634662 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[912f51bc-f762-4485-8a66-863c8b0fff9a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240045, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.654152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c8ade73-fb91-4dfe-9d73-f076769897f3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.678438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b44ddee1-14d5-424c-a942-d6e8d38b0e52]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.699273 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b6dae14-176b-4e3c-9831-b75e83408001]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.720144 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09c8df55-fe27-4458-9429-3963ce9f2f2f]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.720703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:43.720983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:43.721257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:43.721536 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(72868940-f6cd-4159-ba67-e6f48f8fd977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:43.740049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:43.740281 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4bf43ce-33be-4d9f-a2c3-6899400d26da]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.742996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56c5c5f4-1d48-4d03-9ad9-401aa97bfca4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.744647 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-72868940-f6, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:43.745623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Transaction caused no change {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:141}} Jul 01 11:10:43.746124 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-72868940-f6, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:43.746998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:43.747824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:43.748042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:43.748816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-72868940-f6, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '72868940-f6cd-4159-ba67-e6f48f8fd977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:77:ff:0c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:43.795085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29b25e0d-0a47-445d-b7b9-8b39c8cbe3cc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.830247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5f7a09db-28f1-4638-afb6-7fe14b2f35c8]: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:43.830247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:43.830247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:43.830247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:43.830247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:43.830247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:43.830247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:43.830247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:43.830247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:43.830247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:43.830247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:43.830247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:43.830247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:43.831665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f7a09db-28f1-4638-afb6-7fe14b2f35c8]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:43.831863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:43.834162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:43.837008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca14312d-9ab6-4feb-94b7-66d235e308ac]: (4, ['qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.837008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:43.837008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:43.837008 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:43.837419 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:43.848373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c73d46bb-27b4-4838-89b7-0209d7e43a46]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.849865 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ae142e9-f43f-425f-abc6-2ace44572eb7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.850839 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd985ac2-74c3-4ac6-ad35-61078d31afde]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.872773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a944983e-11c1-473c-9ee6-5d42268ce093]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 873989, 'reachable_time': 23628, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240062, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3a979b72-b8c8-4b15-beca-c0617dc5c4f1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-2071907835-router", Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96" Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:43.878428 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:35Z", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:4b:c4", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7659, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:36Z", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72868940-f6cd-4159-ba67-e6f48f8fd977", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a23ef592-5c84-4306-9331-1dbb024be7e9", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:ff:0c", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7660, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:34Z", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:41Z", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id d4f80eb8-f0b6-48e6-8a84-3a3a86a74948. Wait time elapsed: 10.081 Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id d4f80eb8-f0b6-48e6-8a84-3a3a86a74948. Wait time elapsed: 10.082 Jul 01 11:10:43.888181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:44.578013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.700s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:44.578387 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:44.579864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:44.580297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:44.580675 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:44.581935 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12941f91-7183-40a6-aa17-9a2df7e5b7e2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.595557 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5929367e-244c-4b56-8092-efb6b5e231dc]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.617225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ea2df035-2138-4ede-9dd6-a2ce3956734d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.646452 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a52b96e5-6efc-486c-9614-d0b39b5385f9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240066, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.667979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21935734-24c3-4a43-b5d1-21d1c861799a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.689690 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[41417897-a8cc-40c4-8bfc-6ba7ff4b1f32]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.711697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[72f73f3e-2cb3-473a-8b1d-49169f33259f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.732353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[11f9dd07-e90d-442a-b415-c8c7d19527c8]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.732900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:44.733299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:44.733567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:44.733845 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(72868940-f6cd-4159-ba67-e6f48f8fd977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:44.749173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:44.749488 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef008cb6-5a19-42d8-8967-f6bc281eb44f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.752110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a710a0f-7d1d-41a4-801d-a1c4d8de1201]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.753042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-72868940-f6, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:44.789006 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-72868940-f6, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:44.789708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:44.790328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:44.790790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:44.791244 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-72868940-f6, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '72868940-f6cd-4159-ba67-e6f48f8fd977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:77:ff:0c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:44.844279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[289bfd05-8f8e-4cfb-b128-fc5cd3d57e8d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[93270808-7485-42ef-9ead-50a3dbde5fae]: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:44.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:44.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:44.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:44.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:44.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:44.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:44.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:44.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:44.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:44.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:44.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:44.875786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[93270808-7485-42ef-9ead-50a3dbde5fae]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:44.877036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:44.880079 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:44.883718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cfce3c84-5074-4f7c-99b9-99cab4b629bb]: (4, ['qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.883718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:44.883718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:44.883718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:44.883718 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:44.884592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d03ee0b-f727-4046-9770-7549099c6443]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.885916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aedfd425-faee-4842-8f52-5b2c232e8b07]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.887044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86386edc-3fb7-41eb-9428-7237868140e9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.906680 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d560930e-17ca-45b1-8106-01d5c8b344dd]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 874090, 'reachable_time': 25704, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240084, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e118e8ff-c01b-402f-b9bd-0d301174f69f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:44.911855 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-2071907835-router", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96" Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:35Z", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:4b:c4", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7659, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:36Z", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72868940-f6cd-4159-ba67-e6f48f8fd977", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a23ef592-5c84-4306-9331-1dbb024be7e9", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:ff:0c", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7660, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:34Z", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:41Z", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:44.915799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id 981275e7-b76b-4a9f-9e23-d9839b944b6a. Wait time elapsed: 9.040 Jul 01 11:10:44.929561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id 981275e7-b76b-4a9f-9e23-d9839b944b6a. Wait time elapsed: 9.041 Jul 01 11:10:44.929561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:46.424629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.115s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:46.425597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:46.427539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:46.428108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:46.428616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:46.430369 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1e8fe9e9-0a05-43e7-b1f8-e690afdf7e0d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.449940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[78652046-a7b3-4f76-bb7f-b0418706574b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.476941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c6c4a4ca-9b4e-4891-a5e1-7b86274b4f0a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.515905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a12adda5-ece3-461c-9d0d-09db97f1dca6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240087, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.538001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[98862367-e100-47ad-9911-da1f85cbadea]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.561017 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0fa2c767-566d-4f5c-a40e-65ae232fdae7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.584982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d43d189c-7f71-4983-af31-54e28ae327bc]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab1ad775-4b64-4af1-b2d0-d8ff8ffa0783]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.610761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:46.611036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:46.611302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:46.611630 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:46.628801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:46.629645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ffda0b8-bf5b-478b-907e-d40be4e1bf56]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.632321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ca092ac8-ed76-487d-bf96-1318b3f6134b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.633853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:46.681435 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:46.686784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:46.687375 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:46.688024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:46.688486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:46.753191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[137c4dbd-3a14-4721-9184-4caf3cc0fcc2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.789755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[d2768bf2-1f7e-47ac-b3fc-41004f9d5efb]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:46.789755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:46.789755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:46.789755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:46.789755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:46.789755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:46.789755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:46.789755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:46.789755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:46.789755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:46.789755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:46.789755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:46.789755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2768bf2-1f7e-47ac-b3fc-41004f9d5efb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:46.790993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:46.793206 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:46.796645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5a83307-567c-4cd3-bfe1-bfc90a15641b]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.796645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:46.796645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:46.796645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:46.796645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:46.796645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2583e30c-0354-4387-a658-f4b543e06ad2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.798321 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d28fc149-9418-42bd-9812-a804309d8eba]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.798986 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3880dd5c-f95c-4759-ae9a-242a3f73bd9f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.822464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4fa5e446-ca96-4f60-b6e3-66bf183aa3e9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 874278, 'reachable_time': 33976, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240106, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.827953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:46.827953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2bd9fb49-f47b-4abb-9c79-f5ffeab1327c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:46.829266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:42Z", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:46.833562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id c74e4a38-0d8d-4ec1-856f-b0c33399c139. Wait time elapsed: 17.730 Jul 01 11:10:46.849127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id c74e4a38-0d8d-4ec1-856f-b0c33399c139. Wait time elapsed: 17.731 Jul 01 11:10:46.849127 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:47.613010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.782s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:47.614326 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:47.615552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:47.615964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:47.616462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:47.617830 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d7a07f3d-664e-4022-8520-53cdd0cc7ea7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.637948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b4930e99-8599-450c-b81e-c310f651e19f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.662920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c914bf12-3b3a-4e46-b3d8-f4cdc6624b77]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.704719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0e8e6c1f-47fd-4aab-a64b-69c5b6fc4ca0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240110, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.728620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d365f34-7800-4a60-9bf4-837d0f7673b1]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.753639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9a1ad38e-d2e2-4180-915a-99691adc3200]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.778703 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10e6e47f-9add-4ba1-8c57-e1df62b2e556]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.804266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66ed0515-2ad0-4a10-a94e-b0fc5efd9c4e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.805024 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:47.805391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:47.805755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:47.806102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:47.824966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:47.825334 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[433c888a-968c-4044-909d-13a0b8c9710d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.828108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f9d80c3-51ff-49e6-8d99-4541233ff084]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.829201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:47.873035 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:47.873870 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:47.874368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:47.874866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:47.875344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:47.925741 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d23c8ed4-bf8d-47eb-9eda-614501234f90]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.978405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[88c0d2e3-eb15-4a23-9250-2a46b14520e5]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:47.978405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:47.978405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:47.978405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:47.978405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:47.978405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:47.978405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:47.978405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:47.978405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:47.978405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:47.978405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:47.978405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:47.978405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:47.980072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[88c0d2e3-eb15-4a23-9250-2a46b14520e5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:47.980936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:47.984104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:47.987576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87f781fc-1f17-4780-9360-e625cc4b8c46]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.987576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:47.987576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:47.987576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:47.987576 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:47.991762 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d78df23-17a7-4a3e-bfd9-ed97d05c114a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.994423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[82ec4adc-dfe6-4818-94e9-dcc029fd4b58]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:47.995684 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a307c53-8c29-4e5f-a8fa-1b3d86f91a86]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:48.020325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[08ec9519-81f2-4b13-8f7d-2549496c03bb]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 874396, 'reachable_time': 18324, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240130, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:48.032136 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:48.032627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dc27fdbb-1014-4cd3-b30b-f6e5abb7a248]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:48.034133 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:45Z", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:48.039318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 89824402-a4f9-4468-a7f4-3d07796a3b8f. Wait time elapsed: 16.877 Jul 01 11:10:48.063750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 89824402-a4f9-4468-a7f4-3d07796a3b8f. Wait time elapsed: 16.878 Jul 01 11:10:48.063750 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:48.679810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.767s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:48.680391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:48.681262 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:48.681587 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:48.681862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:48.683001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0579a718-f391-4716-b547-0843a045f14a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:48.702469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15095097-0b78-4b6f-b39c-9b55158eef3b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:48.723814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[219731cc-b56d-4aca-b143-3682f9941aa7]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:48.750141 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0975bb6-9650-463f-a864-43fb18d094c0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240195, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:48.772296 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e10fdbf5-b925-474f-a734-ef1bd3c2d342]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:48.792788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fdc6397-ce11-4b2d-9f32-65628df06a4f]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:48.815874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b5e2f4b2-8ace-48c6-832e-6d5fdb615667]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:48.839338 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[525f91ca-cc7b-4151-87bf-990f0014b38e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:48.840012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:48.840293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:48.840560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:48.840843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(72868940-f6cd-4159-ba67-e6f48f8fd977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:48.857300 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:48.857597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[502f8d69-bd4a-456b-a7bd-2c12ea6b84e3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:48.860280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65841b60-05d9-4b5a-a58d-42451d638922]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:48.861227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-72868940-f6, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:48.911793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-72868940-f6, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:48.913010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:48.913607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:48.914629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:48.915344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-72868940-f6, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '72868940-f6cd-4159-ba67-e6f48f8fd977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:77:ff:0c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:49.000549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[717f319c-b5bd-44ed-a5c0-5ae0bbc5fa5d]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.037158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[27e1c45b-9571-47d8-a404-8627310b4dd2]: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:49.037158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:49.037158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:49.037158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:49.037158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:49.037158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:49.037158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:49.037158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:49.037158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:49.037158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:49.037158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:49.037158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:49.037158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27e1c45b-9571-47d8-a404-8627310b4dd2]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:49.038470 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:49.040735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:49.043231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e519817e-0ce4-4468-8e61-1be5f5c7d62f]: (4, ['qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.043231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:49.043231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:49.043231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:49.043231 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:49.043947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a2cde41-2af3-43cc-a305-d3733118c06c]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.044330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cd981a30-b594-4b4b-9d11-faa4c3700282]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.045332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10500864-7b75-445f-b239-1f49b11d7cec]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.066217 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[47cdeddc-c598-4bfa-ad42-55ff4a2b5fa6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 874501, 'reachable_time': 32092, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240213, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.069216 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:49.069681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a9869ad-dcb7-4379-a9b5-4000b7cfaaab]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:49.070993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-2071907835-router", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96" Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:35Z", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:4b:c4", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7659, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:36Z", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72868940-f6cd-4159-ba67-e6f48f8fd977", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a23ef592-5c84-4306-9331-1dbb024be7e9", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:ff:0c", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7660, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:34Z", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:41Z", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:49.076239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:49.090473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id d4f80eb8-f0b6-48e6-8a84-3a3a86a74948. Wait time elapsed: 15.280 Jul 01 11:10:49.090473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id d4f80eb8-f0b6-48e6-8a84-3a3a86a74948. Wait time elapsed: 15.281 Jul 01 11:10:49.090473 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:49.757829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.680s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:49.758443 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:49.759790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:49.760125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:49.760697 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:49.762052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[10cf06c4-925a-43c1-bbf5-c77e4c5f89d1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.777604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4abf3bfc-2855-4f09-be72-cdecc19944e3]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.802101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7edf4607-b543-4d43-9e14-d28c51bacf34]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.836057 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[480e7044-b850-441a-9efe-58d393d4c80b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240216, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.856533 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[790428fd-bc48-403b-a29c-1f1e890b743d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.868793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e65ce065-d0c1-4422-a0ab-5c912fb066b4]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.892642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf88024b-30fa-4d79-a68d-c5429a9e9858]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.914010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61b36870-ac49-4cd9-823a-6ff09da0e25e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.914691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:49.915067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:49.915444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:49.915817 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(72868940-f6cd-4159-ba67-e6f48f8fd977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:49.931909 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:49.932202 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f2f3c8eb-89ff-49a5-96bb-af06a48631a7]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.935055 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c43b4305-ed06-4243-a169-ebc648a73a62]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:49.936042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-72868940-f6, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:49.984081 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-72868940-f6, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:49.984766 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:49.985162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:49.985549 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:49.985934 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-72868940-f6, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '72868940-f6cd-4159-ba67-e6f48f8fd977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:77:ff:0c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:50.053688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[32151b9b-7681-4326-9814-342e468c3c8a]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:50.090446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[b8cc6cb0-08f8-4d35-a48d-7bf59e7a43d3]: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:50.090446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:50.090446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:50.090446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:50.090446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:50.090446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:50.090446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:50.090446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:50.090446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:50.090446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:50.090446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:50.090446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:50.090446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:50.091843 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b8cc6cb0-08f8-4d35-a48d-7bf59e7a43d3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:50.092247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:50.094154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:50.096592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e92b8771-47c3-447b-b40c-696bbc726344]: (4, ['qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:50.096592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:50.096592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:50.096592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:50.096962 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:50.098569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29ad4933-6015-4aa5-af58-820bc2a78723]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:50.102625 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96b777de-c08d-4977-87d1-735131537857]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:50.103791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[18c1960d-d7f9-49ce-b0a7-f152974965f5]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:50.126995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63b22b87-9736-4c6a-81f0-ddc9953c3e90]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 874610, 'reachable_time': 22568, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240234, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:50.139063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:50.139547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fad3106-06be-4f91-a8b6-73fee1775e14]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:50.140727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-2071907835-router", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96" Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:35Z", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:4b:c4", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7659, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:36Z", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72868940-f6cd-4159-ba67-e6f48f8fd977", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a23ef592-5c84-4306-9331-1dbb024be7e9", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:ff:0c", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7660, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:34Z", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:41Z", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:50.144477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:50.165163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id 981275e7-b76b-4a9f-9e23-d9839b944b6a. Wait time elapsed: 14.273 Jul 01 11:10:50.165163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id 981275e7-b76b-4a9f-9e23-d9839b944b6a. Wait time elapsed: 14.274 Jul 01 11:10:50.165163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:51.994648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.959s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:51.995249 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:51.995976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:51.996477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:51.996788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:51.998268 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31baa449-1886-492b-887b-6e7595ed5528]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.011200 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f5187fed-b07b-4787-bf39-1196fd795655]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.034394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8eda57e2-3378-4f0e-89d7-fc2cfe24c8b1]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.063229 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[94ff10da-f653-4599-a0a9-7174715a89d2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240237, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.085577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[694d2999-f9f8-4626-b9ae-ba0732242453]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.106031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1f4abbef-cf99-436c-95c5-9b6dbc53f058]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.128261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdc09c1d-c171-4a8e-b305-51a00c8bfe21]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.150100 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbf54791-2e6d-434b-832b-62c1e8f64a96]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.150664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:52.150940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:52.151312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:52.151754 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:52.167702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:52.168094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[057b70d7-5288-4e06-8f86-a128d6b9935a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.170787 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eba213eb-d02e-4ac5-b329-880c1fff465a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.171802 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:52.214742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:52.215676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:52.216277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:52.217271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:52.218170 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:52.281384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a566e274-ef37-4eac-b4e5-4a65909701e3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.319083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[eee27635-0784-478c-a883-584176967f9b]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:52.319083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:52.319083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:52.319083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:52.319083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:52.319083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:52.319083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:52.319083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:52.319083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:52.319083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:52.319083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:52.319083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:52.319083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:52.320949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eee27635-0784-478c-a883-584176967f9b]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:52.322058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:52.326291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:52.329332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2d15fe7-df68-421f-8eee-dc6e9d96c3e5]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.329332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:52.329586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:52.329687 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:52.330014 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:52.331046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2b074d66-6892-46c3-8078-2ba093687d72]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.332021 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[35d36241-16e8-4d2c-a85a-a664c2d09412]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.333434 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ae822c9-f2dd-4670-98f8-0bc34ddfe1da]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.355114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[821ec040-8ad4-4f22-b4f0-0ff40ee4ee51]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 874832, 'reachable_time': 20876, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240256, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.358691 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:52.359115 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96eb5443-c221-4044-8bb3-8f6e7b877cb5]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:52.361884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:48Z", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:52.366748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id c74e4a38-0d8d-4ec1-856f-b0c33399c139. Wait time elapsed: 23.263 Jul 01 11:10:52.381367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id c74e4a38-0d8d-4ec1-856f-b0c33399c139. Wait time elapsed: 23.263 Jul 01 11:10:52.381367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:53.124287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.978s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:53.125312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:53.126542 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:53.128036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:53.129682 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:53.132788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1fe2ef2c-6e3a-4c71-89a4-45fca98cd4d3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.151396 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bc78bf30-5199-42a1-94f8-927fd57c38b1]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.176332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[74196567-aecd-4a1e-b85c-d8d4232e5f76]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.185099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.822s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:53.185607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:53.186689 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:53.187876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:53.188656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:53.190633 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[00f00bd0-858d-4d53-9da5-a644e866ceec]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.202061 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[58915885-7e4d-492f-a602-34d97adef8d8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.225685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2a0616d4-df2d-481a-bcea-6dbf354a2a23]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240259, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.231560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5cb8a9e1-6fc0-4375-9f3b-690a9727e60f]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.253853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[09db437f-4c73-445d-b082-f7baa2bb614f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.263628 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3fa852f9-48a2-4137-b6cf-321b935a3565]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240264, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.271011 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[51337c62-cc0c-40e0-adf8-6ef8befb436c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.289063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a92fb69-5a70-4504-8b11-dba3d10a95e3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.295768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[be5aaa8b-b38f-4f3e-9b62-8d3611464965]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.302832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb8d0a5b-a079-4764-b0ad-86224ea9d1f9]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.310484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b6fd5ec3-effe-4f47-bbcf-113e042fd748]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.311314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:53.311799 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:53.312130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:53.312597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(72868940-f6cd-4159-ba67-e6f48f8fd977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:53.320820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34324871-a02c-491b-b106-9d0febc9dec2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.338287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:53.338583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8aebd214-3756-4eb9-b6de-5313015453f4]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.341421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63d2c06f-200e-4334-9c3c-e551f42314f9]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.342411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-72868940-f6, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:53.346042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a463535f-29d2-4458-be14-bed5ac754eee]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.350761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:53.351058 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:53.354506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:53.356044 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:53.372179 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:53.372567 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bbb7bae5-bcaa-4fb1-8f8f-b4f0453d3503]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.379582 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d2c8f7d-edc3-4f53-a15c-826f7086a84d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.387534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:53.443858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-72868940-f6, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:53.445654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:53.446123 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:53.446585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:53.447117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-72868940-f6, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '72868940-f6cd-4159-ba67-e6f48f8fd977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:77:ff:0c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:53.509421 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:53.513234 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:53.513977 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:53.514433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:53.514838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:53.524109 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8b41361b-89bb-4544-b0f6-038abcb727d9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.569251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[86e4a2dd-0b3c-4242-b76c-929e313328f8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.579589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4f55d9c2-73cc-4664-9a0b-7dd1bc7b41b2]: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:53.579589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:53.579589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:53.579589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:53.579589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:53.579589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:53.579589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:53.579589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:53.579589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:53.579589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:53.579589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:53.579589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:53.579589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:53.581651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f55d9c2-73cc-4664-9a0b-7dd1bc7b41b2]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:53.583463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:53.587317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:53.591982 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cd74807-bfc7-4eb2-8c6c-0b6098ba8d4b]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.593498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:53.594066 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:53.594517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:53.595089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:53.596990 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bd3e6006-88bf-4c74-977c-75f36e7a09f5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.598886 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e73d66f6-45e4-44e5-bbb5-d2e4c2fb8df2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.603335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a2e4ec8-d07d-45ef-bf04-b5c8c9af5b72]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.611197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[4a9c0eb4-5333-45c4-b034-6c42f760f841]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:53.611197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:53.611197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:53.611197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:53.611197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:53.611197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:53.611197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:53.611197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:53.611197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:53.611197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:53.611197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:53.611197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:53.611197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:53.612341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a9c0eb4-5333-45c4-b034-6c42f760f841]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:53.612890 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:53.615183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:53.617748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6ddeb7e-7ff7-4ec9-a5e1-8e966b9191e7]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.617748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:53.617748 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:53.618134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:53.618134 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:53.620097 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b8f4d59-d672-47ec-b847-cd6a8f13a7ad]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.624659 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c4f26e22-adb1-415f-8e98-4f9d2507cf26]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.625918 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbcddbe8-869f-4a67-88c0-d62ca9bd00de]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.634075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[599441d1-4b10-4e58-8632-a8eade847018]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 874948, 'reachable_time': 31236, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240298, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.640239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:53.640491 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0f7c0df2-24f8-4710-8bb8-f0a74023ced7]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:53.641516 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-2071907835-router", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96" Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:35Z", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:4b:c4", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7659, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:36Z", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72868940-f6cd-4159-ba67-e6f48f8fd977", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a23ef592-5c84-4306-9331-1dbb024be7e9", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:ff:0c", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7660, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:34Z", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:50Z", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 5, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:53.645266 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id d4f80eb8-f0b6-48e6-8a84-3a3a86a74948. Wait time elapsed: 19.847 Jul 01 11:10:53.665023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id d4f80eb8-f0b6-48e6-8a84-3a3a86a74948. Wait time elapsed: 19.847 Jul 01 11:10:53.665023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:53.665023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[73cfaab8-5e93-4229-8aa5-b3540da693ec]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 874952, 'reachable_time': 27088, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240299, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.665023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:53.665023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95e1ca63-e447-4ab8-8d33-ecf9296224ae]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:53.665023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:53.665023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:53.665023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.671735 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:51Z", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 89824402-a4f9-4468-a7f4-3d07796a3b8f. Wait time elapsed: 22.508 Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 89824402-a4f9-4468-a7f4-3d07796a3b8f. Wait time elapsed: 22.508 Jul 01 11:10:53.689459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:54.470453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.804s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:54.471075 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:54.471705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:54.472037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:54.472324 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:54.473692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d71b2abc-e7b8-406e-bb11-c4d7f7b3be83]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.489490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a6afd14-e0b5-4bb6-8110-1d84b0a91c65]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.514770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d122835f-33a8-4d0f-918c-764bcde8be84]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.541152 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[defbdf32-0be0-4448-b6bc-caed6a5f5999]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240303, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.562601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99dab4fc-de27-4deb-9335-6a3a5c44d24b]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.584186 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96bc24ec-b1f1-4476-bbfe-696d2929b03b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.604905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a10cb35-0803-4bb3-854c-1d2be483e348]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.627702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[946b8096-9b33-4189-949f-5b00f515a312]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.628312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:54.628591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:54.628916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:54.630305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:54.645539 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:54.645900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1fd1b18-8baa-4ca3-bfa3-c25a80d4d0ac]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.648665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1526b6c7-e3ca-4543-afc7-a546d8a232cd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.649660 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:54.683545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:54.684524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:54.685015 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:54.685501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:54.685985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:54.756506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2006bdc0-0307-4cc7-9e76-5b85471db929]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.784496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[85d144f2-a1f5-4003-adb1-d16d55a6f28a]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:54.784496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:54.784496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:54.784496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:54.784496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:54.784496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:54.784496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:54.784496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:54.784496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:54.784496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:54.784496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:54.784496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:54.784496 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85d144f2-a1f5-4003-adb1-d16d55a6f28a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:54.785924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:54.787881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:54.790239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6dff669b-e739-4507-8001-f8ff27c928da]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.790239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:54.790239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:54.790239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:54.790239 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:54.790866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15edc717-6781-4b16-85d0-7ac6be0bcc09]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.792197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[140b5784-bf42-4476-af51-ecc019f82b4f]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.793198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eadc609c-80ad-4c90-89cd-5fa8d815637a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.814835 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9d5c9bb-f5d6-45c8-8730-81c8e4700ee6]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 875080, 'reachable_time': 23112, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240321, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.820363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:54.820363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38e5acb1-2b0b-4ef2-9c76-75875944be6f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:54.821471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:51Z", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:54.824747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id c74e4a38-0d8d-4ec1-856f-b0c33399c139. Wait time elapsed: 25.723 Jul 01 11:10:54.838042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id c74e4a38-0d8d-4ec1-856f-b0c33399c139. Wait time elapsed: 25.723 Jul 01 11:10:54.838042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:55.404624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:10:57.611173 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.967s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:57.611844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:57.612791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:57.613303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:57.613825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:57.615171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[944abf4b-660c-4756-93d4-98823f888a40]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.631912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6b32e78-306f-4d9e-a772-7ebf3fcc4e46]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.653544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25ff33e9-6089-4953-82b2-c198d4587e75]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.687649 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[48283190-02a9-4b76-a220-7dc7563a8581]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240385, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.708729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[848658e5-4e68-4acc-88b7-917521fb49ce]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.731224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[393bad27-d3ed-45f4-b8b2-8ae9bed9a947]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.753848 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5e8cfec0-33b9-44b5-a04d-a9a1cd381dc9]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.776153 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8ef5fb12-9dd1-45f5-a3db-54b1a86b8cfc]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.776753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:57.777208 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:57.777481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:57.777755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(72868940-f6cd-4159-ba67-e6f48f8fd977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:57.796091 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:57.796663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4031f480-7c03-451f-9ebb-52e503fb0a04]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.799613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[03467f9f-2f4c-45d5-a96c-e1d463d8e105]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.800808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-72868940-f6, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:57.845484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-72868940-f6, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:57.846523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:57.847137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:57.848552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:57.849161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-72868940-f6, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '72868940-f6cd-4159-ba67-e6f48f8fd977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:77:ff:0c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:57.913874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6eed6058-bc77-4cdb-a649-7ac28b641429]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.956974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[54a3d6f6-60ae-4fd8-8c09-894ec34463bd]: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:57.956974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:57.956974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:57.956974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:57.956974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:57.956974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:57.956974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:57.956974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:57.956974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:57.956974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:57.956974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:57.956974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:57.956974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:57.958764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[54a3d6f6-60ae-4fd8-8c09-894ec34463bd]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:57.959255 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:57.963819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:57.968535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b315a7d-4fed-493c-ac3f-f6f36661986c]: (4, ['qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.969609 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:57.970221 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:57.970654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:57.971156 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:57.974190 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7b26eff8-fec2-47cb-949c-4b08ea5d1f60]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.975588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[494e1e1b-0502-483f-ba3a-cf66dd704433]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:57.976797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1b7b16f-d2ae-4840-9390-dae379d039db]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:58.005901 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0369d5da-608d-4259-91ee-88ff38e4c275]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 875395, 'reachable_time': 30028, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240405, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:58.013027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:58.013232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a38608c1-184c-40cd-ba12-ec254703b92b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:58.014386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-2071907835-router", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96" Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:35Z", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:4b:c4", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7659, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:36Z", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72868940-f6cd-4159-ba67-e6f48f8fd977", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a23ef592-5c84-4306-9331-1dbb024be7e9", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:ff:0c", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7660, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:34Z", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:52Z", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 6, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:58.018225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id 981275e7-b76b-4a9f-9e23-d9839b944b6a. Wait time elapsed: 22.144 Jul 01 11:10:58.032831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id 981275e7-b76b-4a9f-9e23-d9839b944b6a. Wait time elapsed: 22.144 Jul 01 11:10:58.032831 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:58.606598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.783s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:58.607481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:58.609108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:58.609790 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:58.610246 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:58.611532 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f233fcfe-a728-412c-90db-a2494ad94199]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:58.639401 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6e5cf246-1df0-446e-ad0a-c010bd07d595]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:58.676806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[12fac670-1538-4a02-8a44-6c1b15d6f207]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:58.714872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[71b8b3c8-b3af-4d02-8206-e9dcb47b1c69]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240410, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:58.750405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a13154e0-b247-4ce5-a684-84a75fdd4cef]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:58.804613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3b841478-f76d-4420-a410-4e29d2bdf27e]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:58.835768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c59f2281-a3ad-42d9-82ac-5c1ada1cb04b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:58.862884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e9b92c8-2098-49ef-9659-7e897f248408]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:58.863714 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:58.864099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:58.864477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:58.864861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:58.900828 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:58.901259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[697465f2-9bdf-401f-bf3d-2c9dd81fe97a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:58.904433 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f881fd47-8d0e-45d7-91eb-4d833e75c75d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:58.905623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:58.948829 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:58.949566 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:58.950121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:58.950586 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:58.951247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:59.002906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b99c22cc-e2cc-4e06-b9ed-8eae46b463fd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.055665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[56438e58-1b6d-45c8-a15c-f5a1e09d7985]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:59.055665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:59.055665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:59.055665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:59.055665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:59.055665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:59.055665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:59.055665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:59.055665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:59.055665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:59.055665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:59.055665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:59.055665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:59.057376 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56438e58-1b6d-45c8-a15c-f5a1e09d7985]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:59.058556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:59.061688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:59.066168 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc7979e8-5a2b-4b54-8522-e59761560afe]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.067910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:59.068442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:59.068912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:59.069453 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:59.070653 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c256b6b0-7547-4237-863c-96cb74f51ac2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.078184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33433f67-1ed3-4966-a620-a0b8baab0dcd]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.079710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28d20517-5363-4ec7-9667-5caede03720d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.115411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c173583b-ea6e-4872-bc7c-353b4d320b55]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 875497, 'reachable_time': 19132, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240433, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.123420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:59.123780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57da3dbe-a8f3-459c-ae21-a4f2d75d192b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:10:59.125129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:55Z", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 11, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:59.131857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:59.152517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3 Jul 01 11:10:59.152517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 89824402-a4f9-4468-a7f4-3d07796a3b8f. Wait time elapsed: 27.982 Jul 01 11:10:59.152517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 89824402-a4f9-4468-a7f4-3d07796a3b8f. Wait time elapsed: 27.983 Jul 01 11:10:59.152517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:10:59.155937 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.136s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:10:59.156287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:10:59.157442 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:10:59.157858 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:59.158252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:59.162197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27f2bc8a-b6b1-4c62-b115-711649a6052a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.181913 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[28ddca4d-a3da-4532-868d-01bd13e12beb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.208201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-8a70ba3b-14b9-4533-8b6d-9c5fd03a6664 tempest-TestSnapshotPattern-1501344705 tempest-TestSnapshotPattern-1501344705-project] Got routers updated notification :['9b77481e-d5c1-4b97-bc52-fa1e3da962ba'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:10:59.209614 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9652cbee-b058-4d40-a99d-67f3f277bfa8]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.259085 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[295bc7d6-ee28-49e0-8be9-16d43976d7bd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240436, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.298531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f59eccc7-fb47-4145-84dd-cce7fd078ec6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.338276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[20a82042-b947-486e-b548-0808f31accda]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.384974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[811337cd-f080-46ff-b621-6481ad817cc7]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.414849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eace794f-289f-43e5-af78-4b0a3146c278]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.415883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:10:59.416365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:10:59.416834 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:10:59.417343 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(72868940-f6cd-4159-ba67-e6f48f8fd977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:10:59.443386 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:10:59.444374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a660c33c-f638-42b3-a0f9-b5fbf7485052]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.448245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fefa5680-d6cc-44c6-99f0-0576fd791663]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.449362 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-72868940-f6, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:59.496593 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-72868940-f6, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:59.497440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:59.499258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:59.499764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:59.500232 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-72868940-f6, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '72868940-f6cd-4159-ba67-e6f48f8fd977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:77:ff:0c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:10:59.569303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[81f61742-5f9a-4bbe-890a-c6acf96a6c7f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.601727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[e84ee087-b7f1-461c-a9e8-03592cabf466]: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:10:59.601727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:10:59.601727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:10:59.601727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:10:59.601727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:10:59.601727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:10:59.601727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:10:59.601727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:10:59.601727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:10:59.601727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:10:59.601727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:10:59.601727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:10:59.601727 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:59.602877 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e84ee087-b7f1-461c-a9e8-03592cabf466]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:59.602970 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:59.604988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:59.614486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[deba92b3-81fa-42a6-a4b6-02a9f931ba9b]: (4, ['qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.614486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:59.614486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:59.614486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:10:59.614486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:10:59.614486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61970551-5ad4-494e-a861-6db777dd097e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.614486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8911b376-2ae6-4109-9e3a-c0d4e43f0be7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.614486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a13a3f38-9b76-42dc-86c0-ae8974a90e35]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.638302 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[37163647-762b-4890-bc0e-a784ebb4ce04]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 875552, 'reachable_time': 42372, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240455, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.643950 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:10:59.645026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec49985e-9260-46d8-8b47-546dfc3551b0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:10:59.646297 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-2071907835-router", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96" Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:35Z", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:4b:c4", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7659, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:36Z", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72868940-f6cd-4159-ba67-e6f48f8fd977", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a23ef592-5c84-4306-9331-1dbb024be7e9", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:ff:0c", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7660, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:34Z", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:56Z", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 7, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:10:59.668400 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:10:59.690138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id d4f80eb8-f0b6-48e6-8a84-3a3a86a74948. Wait time elapsed: 25.872 Jul 01 11:10:59.690138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id d4f80eb8-f0b6-48e6-8a84-3a3a86a74948. Wait time elapsed: 25.873 Jul 01 11:10:59.690138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:11:00.448222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.779s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:11:00.448497 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:11:00.449996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:11:00.450325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:00.450612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:00.451786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fc7e3f0c-b2a3-4e73-b9e6-ef68fa5dac37]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.464884 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c13448f3-ad9c-4119-9526-e8609007ff2c]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.486330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d2446d19-0c93-43af-ae6a-a7d9213eeaae]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.517010 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f8bed4b5-c3df-4f3c-9f7d-3cfd06c384db]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240459, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.537949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6815670-c359-4df8-a894-dd81156c70ab]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.558503 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0536f38e-8838-4905-baac-1217e0d8f35d]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.580475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2738afee-f27e-48ca-acc2-b63861cc9953]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.601626 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[489a68e0-6fa7-4bf5-9ec5-82d473f48df1]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.602101 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:00.602382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:00.602650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:11:00.602932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(72868940-f6cd-4159-ba67-e6f48f8fd977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:11:00.618832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:11:00.619504 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5367f0b4-59e3-4057-b52f-2f7b6f3e48f2]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.621761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a9b9827c-854c-4e32-ab74-2fbdaf62632d]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.622744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-72868940-f6, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:00.656150 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-72868940-f6, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:00.657225 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:00.657793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:00.658477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:00.659036 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-72868940-f6, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '72868940-f6cd-4159-ba67-e6f48f8fd977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:77:ff:0c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:00.724936 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3ce95d86-c924-4b42-a6dc-f167051817ca]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.755180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c256c8dd-5b5c-4bcd-97d3-f99fa0920ea5]: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:11:00.755180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:11:00.755180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:11:00.755180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:11:00.755180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:11:00.755180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:11:00.755180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:11:00.755180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:11:00.755180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:11:00.755180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:11:00.755180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:11:00.755180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:11:00.755180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:00.756486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c256c8dd-5b5c-4bcd-97d3-f99fa0920ea5]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:00.757135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:00.759639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:00.763280 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb2f722c-211b-4b33-b185-30a8e2db4bbc]: (4, ['qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.764248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:00.764623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:00.764881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:11:00.765291 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:00.766932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8699a453-f5ec-4871-b47b-04db9d3c10ab]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.768166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5a7cfde6-889d-4f72-8217-217f00046dad]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.769117 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6d3f733d-d8f2-49f8-9783-b0b45d51bdc4]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.789174 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0158a91c-dea4-446f-a15c-8d2dfe147a79]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 875678, 'reachable_time': 16700, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240477, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.791194 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:11:00.791664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1ca9021-8494-48cb-94ed-2f74404d6560]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:00.793394 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-2071907835-router", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96" Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:35Z", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:4b:c4", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7659, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:36Z", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72868940-f6cd-4159-ba67-e6f48f8fd977", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a23ef592-5c84-4306-9331-1dbb024be7e9", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:ff:0c", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7660, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:34Z", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:59Z", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 8, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:11:00.797020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id 981275e7-b76b-4a9f-9e23-d9839b944b6a. Wait time elapsed: 24.924 Jul 01 11:11:00.810940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id 981275e7-b76b-4a9f-9e23-d9839b944b6a. Wait time elapsed: 24.924 Jul 01 11:11:00.810940 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:11:02.513377 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.371s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:11:02.513987 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:11:02.514502 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:11:02.514826 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:02.515220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:02.516479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f56c8984-1736-49b5-998f-b1eaa6d6be04]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.533786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40a3387a-a48a-4e31-9056-26c8219a4d57]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.555260 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c856c663-2b45-403d-885e-b921b53f2427]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.585663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b2f829ae-8b74-4373-9c2d-aaa447d9d1d3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240481, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.606850 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8831661d-8797-4ff5-be00-2d485b8b30e4]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.627650 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[21b29f09-da25-4f32-8bc0-d4e5539909d1]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.649370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2388494d-b0af-4d8b-8f07-098271e16a4d]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.669486 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34232f52-209b-4e30-a4bb-9d4faf9cdaaf]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.670140 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:02.670545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:02.670872 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:11:02.671349 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:11:02.695992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:11:02.696692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5da6ea78-ce91-4512-9a55-7de1984cecd9]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.699461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40013a73-be41-43a2-b99f-b81f967f72ad]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.700564 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:02.739895 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:02.740784 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:02.741284 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:02.741938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:02.742552 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:02.803303 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e21d81f4-baee-486d-962a-21411decfad0]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.837082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9aaecc60-791b-40e9-821c-b244e3c6958a]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:11:02.837082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:11:02.837082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:11:02.837082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:11:02.837082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:11:02.837082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:11:02.837082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:11:02.837082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:11:02.837082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:11:02.837082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:11:02.837082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:11:02.837082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:11:02.837082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:02.838113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9aaecc60-791b-40e9-821c-b244e3c6958a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:02.838276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:02.841367 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:02.843966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ae988d1b-cf52-44fa-b717-4bf126969193]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.843966 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:02.844189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:02.844308 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:11:02.844645 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:02.845742 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70f7a1c1-2d4a-4880-9436-166800b12999]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.846722 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a912af6d-b7e9-4a7e-900b-6314420f65a5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.847701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[739799fd-2f89-49b6-95a7-b93ff6f1d9b7]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.875241 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a62a8b5-000e-46fc-9055-b438279200ec]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 875884, 'reachable_time': 42328, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240500, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.878252 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:11:02.878838 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1083861e-b85d-4720-8f31-223d68961377]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:02.881462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "452dc073-00c7-4b85-b44f-f3658383b38f", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.3", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "77621b9e-6b8d-4c59-998a-da10ce3456a5", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7684, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:89:ad:f9", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19064e53-49c5-46f0-a431-a63bbfa5a3de", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:57Z", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:57Z", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:02.886368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:58Z", Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 12, Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3 Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 1, update_id 9f4662ec-a1db-4b83-908b-a417a068a4b4. Wait time elapsed: 3.678 Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 1, update_id 9f4662ec-a1db-4b83-908b-a417a068a4b4. Wait time elapsed: 3.678 Jul 01 11:11:02.901598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:11:03.878469 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.991s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:11:03.880108 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:11:03.882644 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:11:03.886460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:03.887026 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.004s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:03.888537 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[40500e8c-c40f-4013-b87b-fbb4eb11847d]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:03.933798 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2aae61ff-45b5-4b94-ad06-8138419fc747]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:03.981054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f127cfe7-8dc3-4bb4-8386-6b27bc44c036]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.031312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed98050b-54e1-4ce7-aeb1-2d35bd6e9c53]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240548, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.062088 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aebc04cb-71e6-45d1-9e30-d7d3288c3e7a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.081656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.285s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:11:04.082546 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:11:04.083746 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:11:04.084047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:04.084328 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:04.085883 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9066429-66c2-4480-87ed-50f26e16457f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.086445 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2be15f30-2124-4933-bed3-df08411d7d1b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.140801 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5c8bc834-30c4-4b52-8ec6-81d23059c85f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.173820 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36b001df-110f-4277-89a1-7cb576f98cd4]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.188372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f3693b3-d56c-4b87-ace6-b7c1d28c99aa]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.199622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[99acaac8-0d51-444c-8a70-beb4ad660f60]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.199959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:04.200417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:04.202620 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:11:04.203060 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:11:04.245323 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f9a2b4f7-e068-4c11-a0a5-664741630cf7]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240570, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.260258 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:11:04.260821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3d55490e-3d12-4d2e-82b5-3f741c5d44be]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.266465 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[baef3a2c-ab34-4917-87b7-8cd62ca2ad50]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.266756 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b4cb5f4-ac2c-4ae7-862c-809bcbbbe152]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.267808 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:04.298658 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:04.299811 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:04.300275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:04.301271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:04.301721 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:04.308099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1613664c-8112-42c5-a95c-461fad6d372c]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.338031 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ad19ffdd-440e-4180-b91f-24254553c79b]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.369084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d379b92-84ef-4edd-8f3c-aef05ce93580]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.400436 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fdb5012c-628e-4d91-a719-bccd6997e418]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.401000 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:04.401481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:04.403086 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:11:04.403420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(72868940-f6cd-4159-ba67-e6f48f8fd977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:11:04.411238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[0b33e7f4-b288-4a2a-9982-809f738d21d3]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:11:04.411238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:11:04.411238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:11:04.411238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:11:04.411238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:11:04.411238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:11:04.411238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:11:04.411238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:11:04.411238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:11:04.411238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:11:04.411238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:11:04.411238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:11:04.411238 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:04.412372 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0b33e7f4-b288-4a2a-9982-809f738d21d3]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:04.412505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:04.415525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:04.419814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e2927849-15a3-445b-9859-ab1012feb3ff]: (4, ['qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.421513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:04.422004 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:04.422417 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:11:04.422912 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:04.426037 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e4783c63-8c0a-4e37-af3a-3479836f8800]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.427082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:11:04.427551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a865c904-a16f-4af7-8723-7733f8133387]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.427864 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fca1d81e-d8bf-4f0a-9443-49b1b70cce05]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.431293 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66778187-4ef3-4ac7-9498-b98127c19866]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.431747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbf951ec-5831-4fda-ba73-501d972a6b3f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.437335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-72868940-f6, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:04.455764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7bf6879b-fc04-4c68-bdbe-e3c5bf215ca7]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 876029, 'reachable_time': 17180, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240590, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.463505 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:11:04.463764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5065fb28-3397-4b61-aa89-7177d4550202]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:04.464800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "452dc073-00c7-4b85-b44f-f3658383b38f", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.3", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "77621b9e-6b8d-4c59-998a-da10ce3456a5", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7684, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:89:ad:f9", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19064e53-49c5-46f0-a431-a63bbfa5a3de", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:57Z", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:57Z", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:04.469368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:11:00Z", Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 13, Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 9f4662ec-a1db-4b83-908b-a417a068a4b4. Wait time elapsed: 5.257 Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 9f4662ec-a1db-4b83-908b-a417a068a4b4. Wait time elapsed: 5.257 Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-72868940-f6, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:04.489385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-72868940-f6, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '72868940-f6cd-4159-ba67-e6f48f8fd977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:77:ff:0c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:04.527406 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f4369d7-eb2f-4e44-a8c2-87a4970e25a4]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.570639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9e038bfb-492a-4599-8d48-a500db3e2c9d]: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:11:04.570639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:11:04.570639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:11:04.570639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:11:04.570639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:11:04.570639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:11:04.570639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:11:04.570639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:11:04.570639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:11:04.570639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:11:04.570639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:11:04.570639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:11:04.570639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:04.571818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e038bfb-492a-4599-8d48-a500db3e2c9d]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:04.571930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:04.573938 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:04.577446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb1c3048-b42b-4fd5-97c7-b561f03bc519]: (4, ['qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.577446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:04.577446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:04.577446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:11:04.577446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:04.577446 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cf1bfe94-3409-49ca-87b5-33392a9e3cd0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.578201 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bce0477-dc80-48d5-bd22-050b9d1f4d28]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.578979 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f458e9c3-e939-47bd-8db1-39ad3ef1a3b2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.600460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[96f9e66f-2bbd-4b8c-b19c-21109130f3ae]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 876050, 'reachable_time': 44168, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240602, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.602595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:11:04.603125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fa521d45-11af-4f19-9cca-060b3ff6c5db]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:04.604383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-2071907835-router", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96" Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:35Z", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:4b:c4", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7659, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:36Z", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72868940-f6cd-4159-ba67-e6f48f8fd977", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a23ef592-5c84-4306-9331-1dbb024be7e9", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:ff:0c", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7660, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:34Z", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:11:02Z", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 9, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:11:04.607570 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:11:04.624063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id d4f80eb8-f0b6-48e6-8a84-3a3a86a74948. Wait time elapsed: 30.811 Jul 01 11:11:04.624063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id d4f80eb8-f0b6-48e6-8a84-3a3a86a74948. Wait time elapsed: 30.812 Jul 01 11:11:04.624063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:11:05.289984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.681s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:11:05.290591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:11:05.291344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:11:05.291749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:05.292114 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:05.293318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[babdc2d6-fa09-4bb3-a568-fae0e7b6a04e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.307922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[037ff053-ab41-4d58-a462-01be885b10a0]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.329107 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[673cbe77-e870-4016-b625-4e258516dc50]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.356976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d998a0a7-c52a-4abc-bde0-60ad4e961897]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240606, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.377708 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3ae1161-a485-4381-8300-4846cd4cc614]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.399438 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[677e403e-e604-47fe-88eb-90a58ff875e3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.419920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4d6a328c-c98b-4a9a-a3a0-1c77ab5c8944]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.443639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7669f5e6-ee12-47cc-89b9-3f0be150e7bb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.444316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:05.444629 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:05.444902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:11:05.445317 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(72868940-f6cd-4159-ba67-e6f48f8fd977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:11:05.461355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:11:05.461757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4f5d72cb-01e8-4785-b03b-86b4071c3ca1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.464579 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[33f9ddcf-8662-458e-8974-5feb1ea72d26]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.465601 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-72868940-f6, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:05.513836 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-72868940-f6, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:05.514665 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:05.515131 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:05.515753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:05.516463 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-72868940-f6, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '72868940-f6cd-4159-ba67-e6f48f8fd977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:77:ff:0c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:05.570245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[15aec93a-1ca1-4106-a49e-516917cd4aff]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.614254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[1d7568a0-56a2-451f-a4c6-98f30f00279f]: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:11:05.614254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:11:05.614254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:11:05.614254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:11:05.614254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:11:05.614254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:11:05.614254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:11:05.614254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:11:05.614254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:11:05.614254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:11:05.614254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:11:05.614254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:11:05.614254 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1d7568a0-56a2-451f-a4c6-98f30f00279f]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:05.615681 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:05.617933 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:05.621121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e9752443-a760-4a87-9ce7-f0eae217ed02]: (4, ['qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.621121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:05.621121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:05.621121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:11:05.621121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:05.621121 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe214f85-771d-42ec-b52e-c36067f9d4b0]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.621752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1142a721-403a-4631-ade8-ae49056812c2]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.622420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d1fd6508-dc98-4d75-bad6-922511d641f1]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.656350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[26f3f539-a72a-4d07-9f3b-91fd5234425b]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 876162, 'reachable_time': 43340, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240624, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.658444 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:11:05.658857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[68afb8df-fd12-4f47-b72b-796dac5cb287]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:05.660701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-2071907835-router", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96" Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:35Z", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:4b:c4", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7659, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:36Z", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72868940-f6cd-4159-ba67-e6f48f8fd977", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a23ef592-5c84-4306-9331-1dbb024be7e9", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:ff:0c", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7660, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:34Z", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:11:03Z", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:11:05.666534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:11:05.683197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3 Jul 01 11:11:05.683197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id 981275e7-b76b-4a9f-9e23-d9839b944b6a. Wait time elapsed: 29.805 Jul 01 11:11:05.683197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 2, update_id 981275e7-b76b-4a9f-9e23-d9839b944b6a. Wait time elapsed: 29.805 Jul 01 11:11:05.683197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:11:07.976430 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.510s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:11:07.977049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:11:07.978046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:11:07.978374 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:07.978655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:07.979852 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[171bcaaa-83bb-486b-afbc-3428c414c472]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:07.994551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3d5467c-ad24-441b-9347-a5beeda1cb36]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.016382 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[66eeb15b-2f9b-437a-9d1b-f8bc32f1d609]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.045664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a855bed8-8657-48da-8928-54af5f37a29b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240628, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.068158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[312a9eea-6b92-4ede-a8e9-40aba8ff48d7]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.088603 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9ef513b9-1523-4486-ab1e-9255de56724a]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.108598 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e708d6f2-92ee-4988-a7d3-06ee18dfb396]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.137761 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e62c0ca5-19d7-490d-b8a4-5521f9db1b1b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.138318 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:08.138591 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:08.138853 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:11:08.139125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:11:08.154556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:11:08.154851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6a3da2fc-c870-4288-bf58-4461a67418af]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.157753 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8dafc854-0e08-4a7b-9a71-f8b5e049d32a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.158837 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:08.186807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:08.187563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:08.187959 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:08.188341 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:08.188900 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:08.238116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3abebc32-e264-44d7-9b2e-fd0bff9c3ff8]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.272716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c35b4755-3dad-42d1-8b96-66f9d37e5a4a]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:11:08.272716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:11:08.272716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:11:08.272716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:11:08.272716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:11:08.272716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:11:08.272716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:11:08.272716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:11:08.272716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:11:08.272716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:11:08.272716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:11:08.272716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:11:08.272716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:08.274018 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c35b4755-3dad-42d1-8b96-66f9d37e5a4a]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:08.274941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:08.277188 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:08.280604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dbb6de18-9fb8-44a6-80f6-346cb8f86868]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.280604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:08.280604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:08.280604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:11:08.280604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:08.282083 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c42e3530-36a6-437a-b364-59f859bdc108]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.283094 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ffd78909-a701-4d70-bc3d-2570dd97595a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.284106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[34b48096-f66e-4e76-a89e-1b7267e9967c]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.305664 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a8162d22-9f57-4e3d-a59e-5fa8a8a09b50]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 876430, 'reachable_time': 27220, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240647, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ff4d89b-563c-4f09-9a0b-ca3e4c5c9472]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:08.312076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "452dc073-00c7-4b85-b44f-f3658383b38f", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.3", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "77621b9e-6b8d-4c59-998a-da10ce3456a5", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7684, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:89:ad:f9", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19064e53-49c5-46f0-a431-a63bbfa5a3de", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:57Z", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:57Z", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:11:04Z", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:08.318439 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:08.334559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:08.334559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:08.334559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:08.334559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:08.334559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:08.334559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:08.334559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:08.334559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:08.334559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:11:08.334559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:11:08.334559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 9f4662ec-a1db-4b83-908b-a417a068a4b4. Wait time elapsed: 9.101 Jul 01 11:11:08.334559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 9f4662ec-a1db-4b83-908b-a417a068a4b4. Wait time elapsed: 9.102 Jul 01 11:11:08.334559 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:11:08.574793 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 2.897s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:11:08.575180 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:11:08.576743 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:11:08.577139 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:08.577941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:08.579157 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5ed92faf-86a4-4681-bd5e-920830e0103c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.596027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ed3c8bcf-4b7a-4ee7-9093-0f787e581c8b]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.619623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0733b269-8061-417e-91c4-6dbe3d57198d]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.663147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[335ce992-23aa-479f-ac24-b4f6139f00e6]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240650, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.687906 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[740d80c3-f258-4ab0-b61d-27e464989f0f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.710844 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ade5d5e5-e7e2-4987-adcb-e98258e0b4c3]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.737159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[63da595e-7b5a-4bad-8bb1-c0d0a76df4ea]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.761282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0532f444-9826-4b43-90dd-f8cb2571a66b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.762074 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:08.762373 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:08.762642 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:11:08.763054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(72868940-f6cd-4159-ba67-e6f48f8fd977) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:11:08.780612 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:11:08.781693 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[feb5314a-b8ce-452b-985c-2a756de639cc]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.784425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a41a51d4-eed4-4866-800d-89ccd7849861]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.786063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-72868940-f6, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:08.830314 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-72868940-f6, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:08.832967 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:08.834330 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:08.835161 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-72868940-f6, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:08.835786 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-72868940-f6, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '72868940-f6cd-4159-ba67-e6f48f8fd977', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:77:ff:0c'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:08.903857 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1c2ecabe-7a7e-4e25-b523-9f8a8f1780dd]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.952824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[9b0ffecf-8271-4062-afde-eec07a0e6bdb]: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:11:08.952824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:11:08.952824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:11:08.952824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:11:08.952824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:11:08.952824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:11:08.952824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:11:08.952824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:11:08.952824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:11:08.952824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:11:08.952824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:11:08.952824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:11:08.952824 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:08.953946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9b0ffecf-8271-4062-afde-eec07a0e6bdb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:08.954692 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:08.957184 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:08.960604 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6b8c3ff7-8be0-4bea-89b7-1ad52a289988]: (4, ['qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.961440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:08.962178 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:08.962627 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:11:08.963192 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:08.964556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e01e471-b596-4c75-94da-3da34a44d18e]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.965712 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bf38a5c4-8688-4239-9a81-d14330fe7be7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.966955 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec20d090-48f3-4d0b-9f14-8e435585b331]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.994512 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[114c5fcb-2cf8-4569-99a5-036dbba1cf51]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 876492, 'reachable_time': 19416, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240670, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:08.997553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:11:08.998126 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2d4112f8-3be6-40aa-bba5-a39ec3c5a7f8]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-72868940-f6 not found in namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f. Jul 01 11:11:09.002147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-2071907835-router", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96" Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:35Z", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "faff069c-a717-4354-bd44-3b7821b8b0bb", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3b:4b:c4", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7659, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.96", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:32Z", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:36Z", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "72868940-f6cd-4159-ba67-e6f48f8fd977", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "a23ef592-5c84-4306-9331-1dbb024be7e9", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:77:ff:0c", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7660, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:34Z", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:11:03Z", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 10, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "83d0b586e54b42b297f85cc695f5c8e7", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "6f16acfa-3230-45e2-b5d3-78238a4ae4c9", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:11:09.005561 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3 Jul 01 11:11:09.198670 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.888s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:11:09.198994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:11:09.199948 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:11:09.200288 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:09.200548 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:09.202251 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c98518a8-229c-48ef-ade3-a70e8b50a0bb]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.215551 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f49426a7-9222-4b97-b210-e526e0f1e194]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.237248 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1ff22e6-a284-4ace-b279-8a2b7f169547]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.268899 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2f1b5b4f-61ac-4ad9-bd0f-073e1a41f571]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240673, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.292129 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d12d4da4-07a4-4ece-bb64-168563e56eec]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.318841 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7f3538ed-31aa-4315-8bec-397f7b1024be]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.340223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c1798022-9aa8-4ef9-962f-badc436d3344]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.361738 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b600dfb7-c41a-43cc-aea2-be292c75cc0b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.362390 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:09.362656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:09.362951 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:11:09.363340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:11:09.378440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:11:09.379250 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6bfebeef-b237-4b78-9f7c-cd9790e47949]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.382009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bb736840-2617-4119-af39-1733f304041f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.382944 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:09.427263 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:09.428340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:09.428994 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:09.429946 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:09.430674 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:09.571480 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65ea0c1f-ef26-418a-a409-25b0724c2fb9]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.601679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[a5d966c3-21c5-4d43-a5f8-d0b75c09ac30]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:11:09.601679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:11:09.601679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:11:09.601679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:11:09.601679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:11:09.601679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:11:09.601679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:11:09.601679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:11:09.601679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:11:09.601679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:11:09.601679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:11:09.601679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:11:09.601679 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:09.603046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a5d966c3-21c5-4d43-a5f8-d0b75c09ac30]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:09.603585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:09.605988 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:09.609048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[75e26cba-e0e5-4bb9-98c9-b72ea32c08fc]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.609048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:09.609048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:09.609048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:11:09.609048 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:09.610009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5923ee77-693e-45ba-a253-bc2e09047800]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.611067 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d73c931d-f238-4863-ae1a-2de4544205a8]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.612005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[982da639-02a7-4adc-8762-078ae05032ca]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.631632 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9d16e63e-c554-4daa-9c9e-dbdd0c401217]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 876553, 'reachable_time': 27528, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240691, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.639163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:11:09.639528 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[25a77043-90ce-4c9b-952b-c5f13637450f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:09.640678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "452dc073-00c7-4b85-b44f-f3658383b38f", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.3", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "77621b9e-6b8d-4c59-998a-da10ce3456a5", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7684, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:89:ad:f9", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19064e53-49c5-46f0-a431-a63bbfa5a3de", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:57Z", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:57Z", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:09.645046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:11:04Z", Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 14, Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 9f4662ec-a1db-4b83-908b-a417a068a4b4. Wait time elapsed: 10.434 Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 9f4662ec-a1db-4b83-908b-a417a068a4b4. Wait time elapsed: 10.435 Jul 01 11:11:09.659128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:11:13.197295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.552s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:11:13.197916 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:11:13.198492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:11:13.198816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:13.199125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:13.200301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5156cb6d-3abe-4789-b64b-c4608b1839d3]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.216945 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[161eb46a-dff6-43b4-916b-770342f67fbd]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.241095 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9110ee1e-11fb-44d8-a5f5-f4eabbefbd34]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.267910 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c67812fe-96f3-4e8c-a4f9-9ee6bdec2f78]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240757, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.290361 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[882b15ae-ff96-4929-b9c2-496bab546495]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.310822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e7d656f-afad-4eae-9cc2-b295db6680be]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.336875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5d9fb2a9-8df4-4230-a21d-966173053c37]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.358383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[30d6d439-6a0c-4f6d-9f62-3ccf87960d66]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.359016 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:13.359353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:13.359710 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:11:13.360049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:11:13.375437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:11:13.375822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e0bae37f-09f5-4d79-a2cf-8d29c083151a]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.379460 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[856d1924-9cdd-4780-b77d-a82cec1f3240]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.381130 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:13.416733 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:13.418584 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:13.419222 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:13.420418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:13.424747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:13.470914 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fbe1b33d-7012-4995-9d82-6de76cad9ce2]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.506352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[c80221b7-9a75-4d62-ba45-7ba0067d6bf2]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:11:13.506352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:11:13.506352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:11:13.506352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:11:13.506352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:11:13.506352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:11:13.506352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:11:13.506352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:11:13.506352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:11:13.506352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:11:13.506352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:11:13.506352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:11:13.506352 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:13.509331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c80221b7-9a75-4d62-ba45-7ba0067d6bf2]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:13.510513 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:13.513007 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:13.516242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1db07d63-1f23-45b3-8f5e-44828567f9b1]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.516832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:13.517154 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:13.517637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:11:13.517984 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:13.519617 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[acee6a9a-b761-4584-b57a-387524ea1ae9]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.520654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7658289c-ce43-4ad2-9b29-cebe737adaad]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.521615 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[417e4835-4124-4480-9261-1f3237cfc452]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.541729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[442154b2-945c-406f-ab76-6049077a4d2c]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 876953, 'reachable_time': 32172, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240776, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[670a0b0a-1645-4dfe-9a98-3cb0f20b3406]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:13.547148 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "452dc073-00c7-4b85-b44f-f3658383b38f", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.3", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "77621b9e-6b8d-4c59-998a-da10ce3456a5", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7684, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:89:ad:f9", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19064e53-49c5-46f0-a431-a63bbfa5a3de", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:57Z", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:57Z", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:11:08Z", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 15, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:13.554001 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 9f4662ec-a1db-4b83-908b-a417a068a4b4. Wait time elapsed: 14.338 Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 9f4662ec-a1db-4b83-908b-a417a068a4b4. Wait time elapsed: 14.338 Jul 01 11:11:13.578275 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:11:14.288666 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.741s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:11:14.289189 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:11:14.290455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:11:14.291519 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:14.291922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:14.293171 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8f75df9b-9a10-4210-80e2-f8a479a357e1]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.309752 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[084f4545-7e78-411b-8bef-9ac82f8f8565]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.332851 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9e2820b3-10d8-41fe-8ec4-b86e247029aa]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.359607 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7850d0ab-b1de-41d1-a11d-d05302bfa18b]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240780, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.382265 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ec3fb397-ae9e-4d23-90e2-76d6abbd641a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.403585 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[70e0c16d-94c7-4b55-9c02-54f0ab64bc57]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.424183 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[779cbc2c-c63d-43b0-a13b-e05bb5daab4f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.446295 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a3df7c55-21c4-4a7c-a486-9569a3a75e6c]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.446876 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:14.447205 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:14.447531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:11:14.447813 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:11:14.463477 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:11:14.463770 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d44abfa8-da7e-4fb3-9eb7-7cf52cacf378]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.466744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c68e9ad2-af6c-4251-ad13-4dd4781caaf2]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.467739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:14.511789 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:14.512819 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:14.513224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:14.513788 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:14.514365 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:14.570814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60d3c11d-f289-45cf-a57d-19c7866fd007]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.606832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[5f9753f2-2a5f-48be-bf1b-299c1104ee62]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:11:14.606832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:11:14.606832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:11:14.606832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:11:14.606832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:11:14.606832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:11:14.606832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:11:14.606832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:11:14.606832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:11:14.606832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:11:14.606832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:11:14.606832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:11:14.606832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5f9753f2-2a5f-48be-bf1b-299c1104ee62]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:14.608116 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:14.610196 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:14.613668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ee5577a4-fa30-4b53-82b3-b87cc02c52f5]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.613668 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:14.614102 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:14.614441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:11:14.614866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:14.617327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a218afec-1f21-471a-8123-569f136b2f0a]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.617794 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cfcb3bea-6275-4d7e-a27b-dcb080d8ff30]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.618846 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e76c1c3a-8827-40b3-9b9a-ca1c55142e7a]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.639553 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5978ad1f-eaf0-43cb-b88a-d7df68f7e410]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 877062, 'reachable_time': 24668, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240798, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[56df21cb-9c50-4353-a18e-da1f2dd1946f]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:14.647389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "452dc073-00c7-4b85-b44f-f3658383b38f", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.3", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "77621b9e-6b8d-4c59-998a-da10ce3456a5", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7684, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:89:ad:f9", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19064e53-49c5-46f0-a431-a63bbfa5a3de", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:57Z", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:57Z", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:11:11Z", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 16, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:14.653531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:14.669472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:14.669472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:14.669472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:14.669472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:14.669472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:14.669472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:14.669472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:14.669472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:14.669472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:11:14.669472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:11:14.669472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 9f4662ec-a1db-4b83-908b-a417a068a4b4. Wait time elapsed: 15.435 Jul 01 11:11:14.669472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id 9f4662ec-a1db-4b83-908b-a417a068a4b4. Wait time elapsed: 15.435 Jul 01 11:11:14.669472 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:11:15.175042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:15.176110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:15.176734 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:15.176953 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:18.573941 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.929s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:11:18.574534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:11:18.575316 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:11:18.575728 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:18.576072 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:18.577198 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ef8d9055-b63d-48d8-ae19-a2af7d059a6c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.590569 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[27c099b0-9244-4be6-8e38-3ce767f836f6]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.613355 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a72e057e-e659-4479-b40e-bfbdbd3e2d32]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.641311 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d05d640e-70fa-435a-9efe-9e0418ba0b61]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 240865, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.663949 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7a3df3f0-18b1-49ab-a63e-7585b57fe8c6]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.684613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[aaced6ab-bb1c-4b1d-a93a-c187d2476b22]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.705354 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7587c266-a057-462d-8961-c68f168fa106]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.726961 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[65286258-3510-47c8-b270-5587c016139e]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.727739 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:11:18.728137 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:11:18.728498 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:11:18.728849 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:11:18.748924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:11:18.749218 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[bcd40b22-d1fd-4b3c-9010-879c90c85609]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.752012 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[742d37c5-cff8-429c-81e1-f1ccef81e1cd]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.753042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:18.791310 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:18.792047 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:18.792461 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:18.794312 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:18.794729 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:11:18.840092 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b849678c-f365-4ee8-868b-abb79967367c]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.879227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[218bff8d-2b57-4699-9b5e-e3aaf4986a0e]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:11:18.879227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:11:18.879227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:11:18.879227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:11:18.879227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:11:18.879227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:11:18.879227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:11:18.879227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:11:18.879227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:11:18.879227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:11:18.879227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:11:18.879227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:11:18.879227 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:18.880583 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[218bff8d-2b57-4699-9b5e-e3aaf4986a0e]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:18.880686 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:18.882866 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:18.885974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dcdeaecd-b7c6-4b0e-964c-e7c60d16a2e4]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.885974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:18.885974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:18.885974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:11:18.885974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:11:18.886484 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c8d5fa67-8cc8-4914-81be-f0c65d355419]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.887172 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[95a05e40-8974-48ec-8c89-fc8e05ee258d]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.888182 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d3ed0e12-9361-45f4-97fb-2a3eb86eec8f]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.908220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df5ed4ef-3e84-4deb-a291-026d0679b6dc]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 877490, 'reachable_time': 35116, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 240885, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.910163 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:11:18.910437 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8db7b0e0-4256-488d-87bc-a4b2f7a5e168]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:11:18.911657 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_floatingips": [ Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "452dc073-00c7-4b85-b44f-f3658383b38f", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_ip_address": "172.26.5.3", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "floating_network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "router_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_id": "77621b9e-6b8d-4c59-998a-da10ce3456a5", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address": "172.20.0.3", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "DOWN", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7684, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_policy_id": null, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "qos_network_policy_id": null, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_details": { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:89:ad:f9", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "19064e53-49c5-46f0-a431-a63bbfa5a3de", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "compute:nova" Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_domain": "", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_name": "", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:57Z", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:57Z", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 0, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ip_address_scope": null Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:11:18.916385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:11:16Z", Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 17, Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:11:18.929676 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: WARNING neutron.agent.l3.agent [-] Hit retry limit with router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3 Jul 01 11:11:35.406479 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:12:15.176151 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:12:15.176800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:12:15.176800 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:12:15.177002 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:12:15.408087 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:12:55.409821 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:13:15.176413 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:13:15.177158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:13:15.177158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:13:15.177158 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:13:35.411337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:14:15.177104 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:15.177847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:15.177847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:15.177847 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:15.412350 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:14:31.252535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-14131882-b934-4fce-bd1b-546d6407e59b tempest-TestSnapshotPattern-1501344705 tempest-TestSnapshotPattern-1501344705-project] Got routers updated notification :['9b77481e-d5c1-4b97-bc52-fa1e3da962ba'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:14:31.254332 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 1, update_id d09881c1-585a-41f0-b010-97e727d42552. Wait time elapsed: 0.001 Jul 01 11:14:31.254881 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 1, update_id d09881c1-585a-41f0-b010-97e727d42552. Wait time elapsed: 0.001 Jul 01 11:14:31.255384 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:14:32.129905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.875s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:14:32.130242 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:14:32.131289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:14:32.131780 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:32.132128 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:32.133340 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[261e636f-52aa-462e-b7f8-074112936f61]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.146535 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[36cab8f9-8bc2-4697-84ba-fdd8622a2a27]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.168592 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[fe110dc5-36da-4966-bbd6-bbca171c4598]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.198259 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59445e05-a99c-45f2-b441-35bd33eacef3]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 244072, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.219983 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[526345bc-394e-4b51-95ae-d3917973185a]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.240257 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[723f5597-e75d-40cf-8d49-830d99d5911b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.261506 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[42bb3e9b-5a0d-4542-bef6-3a3cb957e1a2]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.282120 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5579e1df-ffd7-4810-93fa-80d60ad4ca73]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.282744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:32.283103 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:32.283464 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:14:32.283749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:14:32.298862 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:14:32.299159 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f098b1a1-2c3e-42f1-b63b-e6ee35f08d60]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.301863 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f21879b4-5c8a-46f6-9a03-abb12ac973b6]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.302875 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:14:32.339902 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:14:32.340575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:14:32.340981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:14:32.341370 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:14:32.341919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:14:32.387764 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0d5e6200-63ff-434d-8bf5-3a664a08c164]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.425737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[8949fd9f-4726-4303-a142-adbc555a6dfb]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:14:32.425737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:14:32.425737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:14:32.425737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:14:32.425737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:14:32.425737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:14:32.425737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:14:32.425737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:14:32.425737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:14:32.425737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:14:32.425737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:14:32.425737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:14:32.425737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:32.426685 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8949fd9f-4726-4303-a142-adbc555a6dfb]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:32.427485 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:32.429143 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:14:32.431327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[83849284-034a-4b2c-a3b2-9e00f29b21bb]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.431327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:14:32.431327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:14:32.431327 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:14:32.431696 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:14:32.434042 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[552cb27f-e4dc-4684-8258-952eb6edd484]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.434998 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[91cbe6c8-ccc4-468d-b6f6-2e80b8a4b116]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.436145 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[c5efb19e-4850-4e54-bf42-d64bd570ee3b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.459353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2c1a7fa-ff10-40d0-81a9-f72010d39296]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 896846, 'reachable_time': 22768, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 244091, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.461305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:14:32.461771 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[716137e4-1cfa-4f9c-91d3-6bbfa08ff3e9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:32.463013 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:11:21Z", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 21, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:14:32.467648 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:14:32.476588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id d09881c1-585a-41f0-b010-97e727d42552. Wait time elapsed: 1.219 Jul 01 11:14:32.476588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 2, update_id d09881c1-585a-41f0-b010-97e727d42552. Wait time elapsed: 1.220 Jul 01 11:14:32.476588 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:14:35.822353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 3.348s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:14:35.822353 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:14:35.823090 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:14:35.823926 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:35.824616 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:35.826256 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[61d10972-a5aa-44e5-a3fe-02043cd59b7e]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:35.839431 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5372589d-d291-44bb-a62e-635c747d1441]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:35.865138 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a98c2fdf-c0ea-4d7d-bbba-a608f99fbc67]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:35.923930 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d4e91980-c218-4eb7-810c-66c9b97bb1dc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 244129, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:35.988391 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2bb08e0e-949c-4926-a4f2-eb4714c38a6d]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.021023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[eb486978-0a5d-4622-8171-4415b1dd26eb]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.048942 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[14238761-66fe-491d-b36e-15d65a2003f8]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.099654 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e3f87e06-d98b-4608-ab60-1a912ccc84eb]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.120466 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-b8e80fc6-e091-4ef4-9ff2-4a73aa021a62 admin admin] Got routers updated notification :['9b77481e-d5c1-4b97-bc52-fa1e3da962ba'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:14:36.121622 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:36.122049 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:36.122544 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:14:36.122958 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] adding internal network: prefix(qr-), port(7abcb02d-c2b0-4cc0-9029-15120e853e74) {{(pid=106231) _internal_network_added /opt/stack/neutron/neutron/agent/l3/router_info.py:548}} Jul 01 11:14:36.160425 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=107952) get_link_id /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:203}} Jul 01 11:14:36.160861 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d11f6643-1a29-47aa-a9a6-3700143bd2d0]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.175039 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e1236a27-44c0-4885-8253-91d69c527bb3]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.185590 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): DelPortCommand(port=qr-7abcb02d-c2, bridge=None, if_exists=True) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:14:36.293507 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=0): AddPortCommand(bridge=br-int, port=qr-7abcb02d-c2, may_exist=False) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:14:36.295082 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=1): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('tag', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:14:36.295954 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=2): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('vlan_mode', 'trunk'),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:14:36.296558 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=3): DbSetCommand(table=Port, record=qr-7abcb02d-c2, col_values=(('trunks', 4095),)) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:14:36.297309 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG ovsdbapp.backend.ovs_idl.transaction [-] Running txn n=1 command(idx=4): DbSetCommand(table=Interface, record=qr-7abcb02d-c2, col_values=(('type', 'internal'), ('external_ids', {'iface-id': '7abcb02d-c2b0-4cc0-9029-15120e853e74', 'iface-status': 'active', 'attached-mac': 'fa:16:3e:c9:09:bd'}))) {{(pid=106231) do_commit /usr/local/lib/python3.8/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py:90}} Jul 01 11:14:36.365492 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[300bfc54-1007-4808-8aa0-52dce4581c22]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 107978, 'error': None, 'target': 'localhost', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.406454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: Exception during request[ab7bbae1-c06b-493a-933c-5120ac6b9423]: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. {{(pid=107978) _process_cmd /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:479}} Jul 01 11:14:36.406454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Traceback (most recent call last): Jul 01 11:14:36.406454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 476, in _process_cmd Jul 01 11:14:36.406454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ret = func(*f_args, **f_kwargs) Jul 01 11:14:36.406454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 274, in _wrap Jul 01 11:14:36.406454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return func(*args, **kwargs) Jul 01 11:14:36.406454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 353, in set_link_attribute Jul 01 11:14:36.406454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: return _run_iproute_link("set", device, namespace, **attributes) Jul 01 11:14:36.406454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 212, in _run_iproute_link Jul 01 11:14:36.406454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: idx = get_link_id(device, namespace) Jul 01 11:14:36.406454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: File "/opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py", line 202, in get_link_id Jul 01 11:14:36.406454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: raise NetworkInterfaceNotFound(device=device, namespace=namespace) Jul 01 11:14:36.406454 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:36.407278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ab7bbae1-c06b-493a-933c-5120ac6b9423]: (5, 'neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound', ('Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.',)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info [-] Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba.: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Traceback (most recent call last): Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return func(*args, **kwargs) Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._process_internal_ports() Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.internal_network_added(p) Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self._internal_network_added(self.ns_name, Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info ns_dev.link.set_mtu(mtu) Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info privileged.set_link_attribute( Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info return self.channel.remote_call(name, args, kwargs, Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info raise exc_type(*result[2]) Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:36.408410 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.router_info Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Error while processing router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:36.409767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:14:36.412220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59b7e51a-a4f1-4957-abae-64e82d9d6c7c]: (4, ['qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'qdhcp-a23ef592-5c84-4306-9331-1dbb024be7e9', 'qdhcp-1a5d3fc9-3dd1-481b-9fbc-c9badbead529', 'qdhcp-893ed888-c195-4963-8385-7fb7f334adce', 'qdhcp-2415b816-8dbb-4efa-8447-4b2949868565', 'qdhcp-afb2987f-4d7f-43f9-b357-669b02318b60', 'qdhcp-5ced1b45-0aba-4ef7-ae4c-f5ebb04f824a', 'qdhcp-5eb98f20-b481-457f-b674-074e10bd84aa', 'qdhcp-14240855-d8fa-49fb-9b3e-e1e45f76b133', 'qdhcp-daa6cca3-c4f0-43b5-ae71-5af24a206e37', 'qdhcp-c369e92b-9b88-40b2-8c7d-00effae68044', 'qdhcp-ebb1c2a9-f7ff-4903-ab4f-5784b66e2708', 'qdhcp-eea6a246-41a4-4c29-9aa5-cba81e33c4e5', 'qdhcp-4e509fcf-4f66-429c-9cb9-2d53c43b2b37', 'qdhcp-07674e54-a5f1-49e1-afb1-ae0823bf359c', 'qdhcp-2b7c4afb-9649-49f5-b98f-7e26bf2b255d', 'qdhcp-15ff82fe-dd1d-4445-a103-1e2c3c6102b5', 'qdhcp-734ab8c1-e6b3-4c59-a80b-0d64c2042a5d', 'qdhcp-a9a659de-f6d1-42d7-9d38-bd056dfcb759', 'qdhcp-50a8c7e7-6a3f-4709-8ced-71586663c300', 'qdhcp-1a59c9af-e8fe-45f5-bb17-f884613de046', 'qdhcp-0fe3c0d2-4579-4662-8752-1dadeffaed17', 'qdhcp-f8fd78c5-9fd8-47d6-a985-6a235f840851', 'qdhcp-11874ba8-fb22-45c0-a33a-5129d93d252b', 'qdhcp-01346cb2-684d-4de2-ba4b-411e5f7a71b7', 'qdhcp-0cb463ae-0884-401c-b0c4-7519eb64a440', 'qdhcp-2bcb40f2-7b62-4be1-960b-d15ec2950831', 'qdhcp-434c7193-1c6c-4a84-b87a-48cc78ea2dd5', 'qdhcp-bf9199e9-5690-40a2-8507-879d803f2772', 'qdhcp-e86068d7-66ec-4e6e-acb9-4c719782c92e', 'qdhcp-f5a34802-2182-4c81-b4ba-6bbec10f68af', 'qdhcp-6d4b0b7e-ba49-4fcc-9362-fc8538061e4d', 'qdhcp-bba55231-77d7-48e6-b0f9-2f2fb7cc1c8f', 'qdhcp-4bc7211b-a193-470d-b88f-69a179daa6b5', 'qdhcp-e68c188b-7855-4484-9148-26099a7fbe2e', 'qdhcp-96b940fe-ca95-4a3b-af66-815a340d9c81', 'qdhcp-1321ae28-b9b5-4d4e-9033-c61d7eafaa7b', 'qdhcp-92fd2355-76b9-4905-99de-c614594c0365', 'qdhcp-e1542b54-457e-4c62-a152-c47080ccbb52', 'qdhcp-a3c5d524-b9f7-4a9a-94cb-f2bafaed74ca', 'qdhcp-46c1954f-b6ea-4704-8bef-168b5666d084', 'qdhcp-218ea180-68a1-4c7b-a6a6-5b57041cd771', 'qdhcp-06998ffc-a5cb-4688-bfb4-6f3bdf0c2ec5', 'qdhcp-85972fb0-baea-4e3e-bdc2-56ba9482b2b9', 'qdhcp-d374f6f6-5a8b-4270-b71a-aafd37f2ab43', 'qdhcp-781390fa-f8e5-4cd9-8b3d-202c0bee6c9e', 'qdhcp-11e947d4-0186-4b65-a6e3-54e8d029c8e1', 'qdhcp-5aa46654-9d32-4a9c-ba93-7c323e01f7cc', 'qdhcp-3e7ca27d-d99e-42f7-bae2-7522be2cedab', 'qdhcp-f2cdda47-bed7-4cb4-beca-94e56a733da1', 'qdhcp-75f6f80b-2ae3-4af4-bf3f-4b099db227ea', 'qdhcp-f952ead8-e792-4f6b-b1d9-292aa643abc4', 'qdhcp-28f402a1-a169-4f42-82fe-5bc39a7145bc', 'qdhcp-70f62913-e0e7-47b3-af9e-ce1bf6f781bd', 'qdhcp-65bdcd4c-c0dd-489c-bc80-44480a44955a', 'qdhcp-a4a83b6a-52b0-4440-b682-75e82dd825c2', 'qdhcp-ecd3bf7b-bf73-4935-9c07-b842725cb3fd', 'qdhcp-117e88dd-6867-4a22-b309-1a5cefe851d5', 'qdhcp-6f7f75d9-308d-4e64-96cf-9c5bed7ffdcd', 'qdhcp-9af24956-4cba-42f8-b5fb-e92e51dc41b6', 'qdhcp-da6c6919-caec-4eb4-9153-b9978cdd14a4', 'qdhcp-cfb7da3c-30fb-425c-850f-d11e1af6f45d', 'qdhcp-bc0fe47c-4ff9-4eb9-adca-e6c805f207ce', 'qdhcp-4ba8af53-14bc-4c8d-bd9c-14ecaa7a8479', 'qdhcp-112a30f7-fa11-4d0d-89b6-e6a344ce22dc', 'qdhcp-72688895-3a47-4e08-b719-2f3703d565c6', 'qdhcp-d7a37a39-831d-48e1-a7e3-b045dceb482e', 'qdhcp-2336f08e-c831-430e-b3bc-8f6ff3ed5ef2', 'qdhcp-ca357faf-b809-4b5b-b070-c6d03078cad8', 'qdhcp-8aaebf27-64f7-4c0c-bf05-2cf0a3f018b9', 'qdhcp-c21f3643-3e41-4db0-97a0-4a6ecc213ff8', 'qdhcp-defa5557-8412-4ed1-ab97-912701c7a4c4', 'qdhcp-2b71076a-940c-4868-a739-6596faed3a10']) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.412220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:14:36.412220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:14:36.412220 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No haproxy process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:14:36.412511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:14:36.413383 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e8507ec-508e-49d0-acd4-3d43258958cf]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.414556 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4a04ee9f-3830-418d-a1d8-70ecbcbaad14]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.423457 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[16839fa3-428c-408d-9038-a61c4d434e1b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.468702 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[62fa6b31-e2e4-437b-b1bf-7e03463ba6b8]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 897218, 'reachable_time': 29064, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 244182, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.472924 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:14:36.473181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb8d5339-d8cb-4dc1-afb4-0d3039f51b20]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent [-] Failed to process compatible router: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba: neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Traceback (most recent call last): Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 848, in _process_routers_if_compatible Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_router_if_compatible(router) Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 635, in _process_router_if_compatible Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_added_router(router) Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 656, in _process_added_router Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._cleanup_failed_router(router['id'], Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/agent.py", line 646, in _process_added_router Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ri.process() Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 178, in call Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.logger(e) Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 227, in __exit__ Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.force_reraise() Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line 200, in force_reraise Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise self.value Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/common/utils.py", line 175, in call Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return func(*args, **kwargs) Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 1302, in process Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._process_internal_ports() Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 688, in _process_internal_ports Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.internal_network_added(p) Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 570, in internal_network_added Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self._internal_network_added(self.ns_name, Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/l3/router_info.py", line 550, in _internal_network_added Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.driver.plug(network_id, port_id, interface_name, mac_address, Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 267, in plug Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.plug_new(network_id, port_id, device_name, mac_address, Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 433, in plug_new Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent self.set_mtu(device_name, mtu, namespace=namespace, prefix=prefix) Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/interface.py", line 473, in set_mtu Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent ns_dev.link.set_mtu(mtu) Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/opt/stack/neutron/neutron/agent/linux/ip_lib.py", line 450, in set_mtu Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent privileged.set_link_attribute( Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/priv_context.py", line 271, in _wrap Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent return self.channel.remote_call(name, args, kwargs, Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent File "/usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py", line 215, in remote_call Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent raise exc_type(*result[2]) Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent neutron.privileged.agent.linux.ip_lib.NetworkInterfaceNotFound: Network interface qr-7abcb02d-c2 not found in namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba. Jul 01 11:14:36.474402 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ERROR neutron.agent.l3.agent Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Payload: Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "tempest-TestSnapshotPattern-933823322-router", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_gateway_info": { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "external_fixed_ips": [ Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76" Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zones": [ Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "nova" Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "distributed": false, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha": false, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ha_vr_id": 0, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "availability_zone_hints": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "routes": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "flavor_id": null, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:30Z", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port": { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "372fc1a5-70de-43b9-bc53-1d80d2092369", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "3978e771-1614-417d-ac17-f2ea7e60d8e3", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:3e:92:99", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_gateway", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7642, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.26.5.76", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 24 Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:27Z", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:10:33Z", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 4, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "5349a4bf-485c-463a-bf8f-e3babc2dec02", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.26.5.0/24", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.26.5.1", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "enable_snat": true, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gw_port_host": "dv-neutron-848393-1-hyperv", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "_interfaces": [ Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "7abcb02d-c2b0-4cc0-9029-15120e853e74", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "name": "", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "network_id": "1a5d3fc9-3dd1-481b-9fbc-c9badbead529", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tenant_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mac_address": "fa:16:3e:c9:09:bd", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "admin_state_up": true, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "status": "ACTIVE", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_id": "9b77481e-d5c1-4b97-bc52-fa1e3da962ba", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "device_owner": "network:router_interface", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "standard_attr_id": 7646, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "fixed_ips": [ Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnet_id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ip_address": "172.20.0.1", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "prefixlen": 28 Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "allowed_address_pairs": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_dhcp_opts": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "security_groups": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "description": "", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vnic_type": "normal", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:profile": {}, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:host_id": "dv-neutron-848393-1-hyperv", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_type": "ovs", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "binding:vif_details": { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "connectivity": "l2", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_filter": true, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ovs_hybrid_plug": false, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "datapath_type": "system", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bridge_name": "br-int", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "bound_drivers": { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "0": "openvswitch" Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "port_security_enabled": false, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "tags": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "created_at": "2022-07-01T11:10:29Z", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "updated_at": "2022-07-01T11:11:21Z", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "revision_number": 21, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "project_id": "d5fdde4430fc4099ae8902c4257dfc74", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnets": [ Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "id": "116fa2ad-6ead-4958-8c30-e30d729dad8d", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "cidr": "172.20.0.0/28", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "gateway_ip": "172.20.0.1", Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "dns_nameservers": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "ipv6_ra_mode": null, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "subnetpool_id": null Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "extra_subnets": [], Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "address_scopes": { Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "4": null, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "6": null Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: }, Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: "mtu": 1500 Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: ] Jul 01 11:14:36.478947 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: } {{(pid=106231) log_verbose_exc /opt/stack/neutron/neutron/agent/l3/agent.py:92}} Jul 01 11:14:36.491581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 1, update_id 5da65a6c-15cd-4a89-a1e3-fa38de72e1f2. Wait time elapsed: 0.368 Jul 01 11:14:36.491581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 3, priority 1, update_id 5da65a6c-15cd-4a89-a1e3-fa38de72e1f2. Wait time elapsed: 0.368 Jul 01 11:14:36.491581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:14:37.040106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 0.550s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:14:37.040705 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:14:37.041420 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:14:37.042023 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:37.042493 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:37.043964 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9016f460-eeac-40a1-a437-f70c973b8c8c]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.060483 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a975b740-0aff-4e2b-af31-ea70eaafd506]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.084543 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a687f7bd-8f59-4c56-a81e-cf0403114268]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.123511 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[dd2a9d2f-f993-4681-a233-466b689fab6f]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 244211, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.146175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a2da6c9b-01db-4fca-a797-a2b2d0f2310f]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.167892 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[29ebafa4-20aa-4d15-86f5-6dd5a1fba4e7]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.195224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[061cf51b-aebf-452d-9022-914e50ead08f]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.220547 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1587ecc8-85df-4bbe-a019-6be4ae31033b]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.220810 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:37.221304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:37.221624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:14:37.246363 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13fc6251-0447-4403-a1f5-846a6d7cf2f9]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 897338, 'reachable_time': 42424, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 244217, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.247267 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:37.247782 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:37.248418 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:14:37.248773 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" acquired by "process_external" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 11:14:37.276271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[913a8db5-d750-4211-b47d-339fab3894d3]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 897338, 'reachable_time': 42424, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 244218, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.278071 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:14:37.278623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 11:14:37.304526 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6954d7e0-f7a3-4784-9b97-4dd455b8e310]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.394624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-6a9e6534-ccf2-445e-9083-2857641a180b admin admin] Got router deleted notification for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 11:14:37.494868 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abb30b44-ec34-46fd-9477-5351a0d61e90]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.510981 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7d9d6ac8-47e3-4370-b04b-6d34eebf9909]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.735106 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[784ccc62-4b89-4d84-8f9c-9a1367e71c86]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.735678 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 11:14:37.735993 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:14:37.736276 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:14:37.736806 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" released by "process_external" :: held 0.488s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 11:14:37.737325 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:14:37.737651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 11:14:37.738223 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:14:37.738905 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 11:14:37.754920 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[85ab0503-5262-4fa9-83a2-039f73f531f2]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.790563 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[13858b55-6b31-4126-b2a6-50a152fc381e]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.821772 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3cced003-20d6-44db-a054-61325cfacf01]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.859210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[da821345-ff4d-46cd-b3e3-d4c6a23eba74]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.860514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 11:14:37.860963 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:14:37.862009 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:14:37.862399 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" released by "process_address_scope" :: held 0.125s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 11:14:37.862874 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:14:37.863471 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:14:37.864608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" acquired by "apply_metadata_nat_rules" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 11:14:37.867701 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:14:37.868707 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 11:14:37.902289 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[ccc255b3-bb65-42ca-8ba2-ad7f53ea476a]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.972825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a4ad7968-d6f3-4ea5-882e-13738ee8ad98]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:37.994046 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[17e9ef93-61d9-4cad-9c69-661e33495b56]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:37 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 11:14:37 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:38.067054 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f1aa2254-9e22-4525-8b1d-cfc7f9a10e76]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:38.068261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 11:14:38.068767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:14:38.069197 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:14:38.069768 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba" released by "apply_metadata_nat_rules" :: held 0.206s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 11:14:38.070432 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-9b77481e-d5c1-4b97-bc52-fa1e3da962ba Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.haproxy Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID 9b77481e-d5c1-4b97-bc52-fa1e3da962ba Jul 01 11:14:38.071135 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 11:14:38.074613 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 11:14:38.101586 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-9b77481e-d5c1-4b97-bc52-fa1e3da962ba[244234]: Proxy listener started. Jul 01 11:14:38.104778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 11:14:38.105510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, update_id 5da65a6c-15cd-4a89-a1e3-fa38de72e1f2. Time elapsed: 1.616 Jul 01 11:14:38.106278 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 1, priority 1, update_id b36a9f2a-10fc-4aa0-aee7-4c8c0d08098d. Wait time elapsed: 0.711 Jul 01 11:14:38.106534 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, action 1, priority 1, update_id b36a9f2a-10fc-4aa0-aee7-4c8c0d08098d. Wait time elapsed: 0.712 Jul 01 11:14:38.106832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 11:14:38.107149 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:14:38.118651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[59cfd23f-f892-46d1-a578-a5faa867ff66]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:38.120575 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[f61e80d4-6e05-4401-8058-2af8aa678d41]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:38.121572 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[0dc22b51-a6f6-4492-bb2c-2b38b73b13a7]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:38.122240 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 11:14:38.123287 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[53779233-5038-4e6e-b239-de9803d05f91]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:38.144247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[df54eb13-86df-45d9-ab3b-eafc6f33ae53]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 897338, 'reachable_time': 42424, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 244240, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:38.146050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:38.146050 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:38.167974 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[9cdeeb30-cd2a-481f-adeb-adb5e611c505]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 897338, 'reachable_time': 42424, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 244241, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:38.168716 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 11:14:38.169214 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:14:38.169550 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/9b77481e-d5c1-4b97-bc52-fa1e3da962ba.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:14:38.169791 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:14:38.171335 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router 9b77481e-d5c1-4b97-bc52-fa1e3da962ba {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 11:14:38.173767 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[092db752-7b43-4afa-b2f4-7f21cf70ee39]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:38.192481 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[2397c5b2-8fe6-4b99-8573-4fe1e4baee1f]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 897338, 'reachable_time': 42424, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 244242, 'error': None, 'target': 'qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:38.199989 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-9b77481e-d5c1-4b97-bc52-fa1e3da962ba deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:14:38.200277 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[87a0cc66-64f1-4b68-984f-05aaf7cec3a9]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:38.200704 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info 9b77481e-d5c1-4b97-bc52-fa1e3da962ba delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 11:14:38.201040 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (9b77481e-d5c1-4b97-bc52-fa1e3da962ba), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:14:38.201385 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:38.201663 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:38.202514 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:38.203545 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:38.203818 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for 9b77481e-d5c1-4b97-bc52-fa1e3da962ba, update_id b36a9f2a-10fc-4aa0-aee7-4c8c0d08098d. Time elapsed: 0.098 Jul 01 11:14:39.412162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-01de49fe-94b8-4bc6-aa0e-357c501d3a87 admin admin] Got routers updated notification :['a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f'] {{(pid=106231) routers_updated /opt/stack/neutron/neutron/agent/l3/agent.py:579}} Jul 01 11:14:39.413195 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 1, update_id ef397daa-5e5f-43ad-9ed4-4f58dee8981d. Wait time elapsed: 0.001 Jul 01 11:14:39.416279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 3, priority 1, update_id ef397daa-5e5f-43ad-9ed4-4f58dee8981d. Wait time elapsed: 0.001 Jul 01 11:14:39.416279 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers start {{(pid=106231) wrapper /opt/stack/neutron/neutron/common/utils.py:935}} Jul 01 11:14:40.684093 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [None req-c472f250-e982-4bc9-a621-3d60d9e06629 admin admin] Got router deleted notification for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) router_deleted /opt/stack/neutron/neutron/agent/l3/agent.py:571}} Jul 01 11:14:40.948922 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.utils [-] Time-cost: call 5a0e6787-e05b-447a-bc0c-5b6ab22188b1 function get_routers took 1.535s seconds to run {{(pid=106231) wrapper /usr/local/lib/python3.8/dist-packages/oslo_utils/timeutils.py:322}} Jul 01 11:14:40.949441 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f info not in cache, will do the router add action. {{(pid=106231) _process_router_if_compatible /opt/stack/neutron/neutron/agent/l3/agent.py:633}} Jul 01 11:14:40.951099 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.linux.pd.add_router-8767249872357'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:14:40.951623 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.add_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:40.953822 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.add_router" :: held 0.002s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:40.955608 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[cb5ffc4b-d822-4e39-aaed-59af0110872f]: (4, False) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:40.974076 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4ad63201-6391-45b6-8d05-0a77f8a567eb]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:40.999062 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[38a6fd8e-1408-4020-a19a-4e43f694ec9a]: (4, ('net.ipv4.conf.all.promote_secondaries = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.028269 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b99cccc0-0bb2-4187-968f-3caf8700bfcc]: (4, ({'header': {'length': 36, 'type': 2, 'flags': 256, 'sequence_number': 255, 'pid': 244298, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': Stats(qsize=0, delta=0, delay=0)}, 'event': 'NLMSG_ERROR'},)) {{(pid=107978) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.050581 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[731f9417-1fbf-48a6-83b7-b30dc38f2ff3]: (4, ('net.ipv4.ip_forward = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.071715 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[4739c4f6-2edd-475a-81e9-b3974e52da9b]: (4, ('net.ipv4.conf.all.arp_ignore = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.094807 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6c4523b0-31ac-4393-b4ec-a7ec5ca675fd]: (4, ('net.ipv4.conf.all.arp_announce = 2\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.117524 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5fc54a03-2b1e-4fc3-abed-96a28022bce3]: (4, ('net.ipv6.conf.all.forwarding = 1\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.118005 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:41.118378 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:41.118792 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process updates, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process /opt/stack/neutron/neutron/agent/l3/router_info.py:1299}} Jul 01 11:14:41.141447 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[60d24a50-7b9e-4894-9996-48c6b12580a2]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 897729, 'reachable_time': 41896, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 244304, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.142651 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:41.142921 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:41.143348 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:14:41.144531 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" acquired by "process_external" :: waited 0.001s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 11:14:41.167053 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a8df003-e996-40bf-8479-4ed7d6b14a73]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 897729, 'reachable_time': 41896, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 244305, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.169020 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:14:41.169589 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 11:14:41.183282 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[e860f1f9-801d-4a7a-9397-f5f317599ec3]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.354637 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[5dc0e794-48f4-4192-acba-886ee0ef29e8]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.375166 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1a120519-4cfd-4b17-b46a-d3a02174e6ae]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.566996 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[a6bbfa6e-c37a-456d-8c55-cbc99fe3e0a6]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.567577 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 140 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 11:14:41.567992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:14:41.568344 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:14:41.568694 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" released by "process_external" :: held 0.425s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 11:14:41.569245 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:14:41.569560 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" acquired by "process_address_scope" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 11:14:41.569995 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:14:41.570631 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 11:14:41.616594 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[22be3564-8131-49ce-817f-ca26c31dc89f]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.650597 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[abfdaf0a-2572-4a45-af59-f556b5d969f3]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.678778 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[291ba446-397e-46b3-9e4e-0571d8cf12f2]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\n-A neutron-l3-agent-scope -m mark --mark 0x1/0xffff -j DROP\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.729510 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[6533de50-b401-460c-8e29-4c67eab86997]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.730125 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 10 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 11:14:41.730304 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:14:41.730655 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:14:41.730869 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" released by "process_address_scope" :: held 0.161s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 11:14:41.731247 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.after_router_added-8767249872954'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), after_create {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:14:41.731744 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:14:41.731999 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" acquired by "apply_metadata_nat_rules" :: waited 0.000s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:73}} Jul 01 11:14:41.732517 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired lock "iptables-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:294}} Jul 01 11:14:41.733737 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Acquired external semaphore "iptables-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:302}} Jul 01 11:14:41.750423 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[84cc7817-91f6-4656-aeb3-2e53d8f9c527]: (4, ('# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-snat - [0:0]\n:neutron-postrouting-bottom - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A POSTROUTING -j neutron-postrouting-bottom\n-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat\n-A neutron-postrouting-bottom -m comment --comment "Perform source NAT on outgoing traffic." -j neutron-l3-agent-snat\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-float-snat - [0:0]\n:neutron-l3-agent-floatingip - [0:0]\n:neutron-l3-agent-mark - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-mark\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-floatingip\n-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\n-A neutron-l3-agent-float-snat -m connmark --mark 0x0/0xffff0000 -j CONNMARK --save-mark --nfmask 0xffff0000 --ctmask 0xffff0000\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n# Generated by iptables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.806885 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[31a5585c-5cd7-4d02-ba6c-6fb2d95560c2]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.822985 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57ce9426-b11f-4390-81ea-674d44832906]: (4, ('# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*raw\n:PREROUTING ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A OUTPUT -j neutron-l3-agent-OUTPUT\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*nat\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*mangle\n:PREROUTING ACCEPT [0:0]\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:POSTROUTING ACCEPT [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-POSTROUTING - [0:0]\n:neutron-l3-agent-PREROUTING - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A PREROUTING -j neutron-l3-agent-PREROUTING\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A POSTROUTING -j neutron-l3-agent-POSTROUTING\n-A neutron-l3-agent-PREROUTING -j neutron-l3-agent-scope\n-A neutron-l3-agent-PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000\n-A neutron-l3-agent-PREROUTING -d fe80::a9fe:a9fe/128 -i qr-+ -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x1/0xffff\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n# Generated by ip6tables-save v1.8.4 on Fri Jul 1 11:14:41 2022\n*filter\n:INPUT ACCEPT [0:0]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [0:0]\n:neutron-filter-top - [0:0]\n:neutron-l3-agent-FORWARD - [0:0]\n:neutron-l3-agent-INPUT - [0:0]\n:neutron-l3-agent-OUTPUT - [0:0]\n:neutron-l3-agent-local - [0:0]\n:neutron-l3-agent-scope - [0:0]\n-A INPUT -j neutron-l3-agent-INPUT\n-A FORWARD -j neutron-filter-top\n-A FORWARD -j neutron-l3-agent-FORWARD\n-A OUTPUT -j neutron-filter-top\n-A OUTPUT -j neutron-l3-agent-OUTPUT\n-A neutron-filter-top -j neutron-l3-agent-local\n-A neutron-l3-agent-FORWARD -j neutron-l3-agent-scope\nCOMMIT\n# Completed on Fri Jul 1 11:14:41 2022\n', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.879299 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1aace08d-7917-4e68-b68c-56ad137da148]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.882224 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.iptables_manager [-] IPTablesManager.apply completed with success. 22 iptables commands were issued {{(pid=106231) _apply_synchronized /opt/stack/neutron/neutron/agent/linux/iptables_manager.py:625}} Jul 01 11:14:41.882816 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "iptables-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:14:41.883271 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Releasing lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" {{(pid=106231) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:312}} Jul 01 11:14:41.884089 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.common.coordination [-] Lock "router-lock-ns-qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f" released by "apply_metadata_nat_rules" :: held 0.152s {{(pid=106231) _synchronized /opt/stack/neutron/neutron/common/coordination.py:85}} Jul 01 11:14:41.884814 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.metadata.driver [-] haproxy_cfg = Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: global Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log /dev/log local0 debug Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log-tag haproxy-metadata-proxy-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: user ubuntu Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: group ubuntu Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: maxconn 1024 Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: pidfile /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.haproxy Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: daemon Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: defaults Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: log global Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: mode http Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option httplog Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option dontlognull Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option http-server-close Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: option forwardfor Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: retries 3 Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-request 30s Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout connect 30s Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout client 32s Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout server 32s Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: timeout http-keep-alive 30s Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: listen listener Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: bind :::9697 Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: server metadata /opt/stack/data/neutron/metadata_proxy Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request del-header X-Neutron-Network-ID Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: http-request set-header X-Neutron-Router-ID a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f Jul 01 11:14:41.885455 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: {{(pid=106231) create_config_file /opt/stack/neutron/neutron/agent/metadata/driver.py:161}} Jul 01 11:14:41.888162 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'haproxy', '-f', '/opt/stack/data/neutron/ns-metadata-proxy/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.conf'] {{(pid=106231) execute_rootwrap_daemon /opt/stack/neutron/neutron/agent/linux/utils.py:108}} Jul 01 11:14:41.924046 dv-neutron-848393-1-hyperv haproxy-metadata-proxy-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f[244319]: Proxy listener started. Jul 01 11:14:41.926719 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f add action. {{(pid=106231) add_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:58}} Jul 01 11:14:41.927747 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, update_id ef397daa-5e5f-43ad-9ed4-4f58dee8981d. Time elapsed: 2.514 Jul 01 11:14:41.928084 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting processing update a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 1, priority 1, update_id d53a3466-d95a-489b-b688-de77e447dc31. Wait time elapsed: 1.243 Jul 01 11:14:41.928490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Starting router update for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, action 1, priority 1, update_id d53a3466-d95a-489b-b688-de77e447dc31. Wait time elapsed: 1.244 Jul 01 11:14:41.928923 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.l3_agent_extensions_manager [-] L3 agent extension(s) finished router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f delete action. {{(pid=106231) delete_router /opt/stack/neutron/neutron/agent/l3/l3_agent_extensions_manager.py:72}} Jul 01 11:14:41.929389 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.metadata.driver.before_router_removed-8767249873080'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), before_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:14:41.972688 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[57fd8b90-17c0-4e71-9856-dc3fcbd8343b]: (4, ('', '', 0)) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.974301 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[45a86d43-0aaa-4872-a950-3132637f04a5]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.976624 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[1b9a5277-c085-4b8e-a0d8-aac23844e0db]: (4, None) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:41.977147 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Process delete, router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) process_delete /opt/stack/neutron/neutron/agent/l3/router_info.py:1288}} Jul 01 11:14:41.977992 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[8e3d76cd-c2bc-4569-8ebd-3dd1540522df]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:42.005490 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[d6f4b791-39a5-461a-a292-f7bbc8f94d71]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 897729, 'reachable_time': 41896, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 244327, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:42.006595 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:42.007027 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.PrefixDelegation.sync_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:42.028210 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[b9694d19-f6d8-4c70-abfe-51c5fd4c82b5]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 897729, 'reachable_time': 41896, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 244328, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:42.028932 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.router_info [-] Terminating radvd daemon in router device: a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable_radvd /opt/stack/neutron/neutron/agent/l3/router_info.py:640}} Jul 01 11:14:42.029422 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:14:42.029774 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.utils [-] Unable to access /opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.radvd; Error: [Errno 2] No such file or directory: '/opt/stack/data/neutron/external/pids/a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f.pid.radvd' {{(pid=106231) get_value_from_file /opt/stack/neutron/neutron/agent/linux/utils.py:252}} Jul 01 11:14:42.030113 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.external_process [-] No radvd process started for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/external_process.py:125}} Jul 01 11:14:42.031405 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.linux.ra [-] radvd disabled for router a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f {{(pid=106231) disable /opt/stack/neutron/neutron/agent/linux/ra.py:200}} Jul 01 11:14:42.032426 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[52e416b6-6c07-4cb9-a349-29206942bd1b]: (4, True) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:42.053078 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[7e9cbed3-4503-40dd-8a4e-56d28e5a3478]: (4, [{'family': 0, '__align': (), 'ifi_type': 772, 'index': 1, 'flags': 65609, 'change': 0, 'attrs': [['IFLA_IFNAME', 'lo'], ['IFLA_TXQLEN', 1000], ['IFLA_OPERSTATE', 'UNKNOWN'], ['IFLA_LINKMODE', 0], ['IFLA_MTU', 65536], ['IFLA_MIN_MTU', 0], ['IFLA_MAX_MTU', 0], ['IFLA_GROUP', 0], ['IFLA_PROMISCUITY', 0], ['IFLA_NUM_TX_QUEUES', 1], ['IFLA_GSO_MAX_SEGS', 65535], ['IFLA_GSO_MAX_SIZE', 65536], ['IFLA_NUM_RX_QUEUES', 1], ['IFLA_CARRIER', 1], ['IFLA_QDISC', 'noqueue'], ['IFLA_CARRIER_CHANGES', 0], ['IFLA_PROTO_DOWN', 0], ['IFLA_CARRIER_UP_COUNT', 0], ['IFLA_CARRIER_DOWN_COUNT', 0], ['IFLA_MAP', {'mem_start': 0, 'mem_end': 0, 'base_addr': 0, 'irq': 0, 'dma': 0, 'port': 0}], ['IFLA_ADDRESS', '00:00:00:00:00:00'], ['IFLA_BROADCAST', '00:00:00:00:00:00'], ['IFLA_STATS64', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_STATS', {'rx_packets': 0, 'tx_packets': 0, 'rx_bytes': 0, 'tx_bytes': 0, 'rx_errors': 0, 'tx_errors': 0, 'rx_dropped': 0, 'tx_dropped': 0, 'multicast': 0, 'collisions': 0, 'rx_length_errors': 0, 'rx_over_errors': 0, 'rx_crc_errors': 0, 'rx_frame_errors': 0, 'rx_fifo_errors': 0, 'rx_missed_errors': 0, 'tx_aborted_errors': 0, 'tx_carrier_errors': 0, 'tx_fifo_errors': 0, 'tx_heartbeat_errors': 0, 'tx_window_errors': 0, 'rx_compressed': 0, 'tx_compressed': 0}], ['IFLA_XDP', {'attrs': [['IFLA_XDP_ATTACHED', None]]}], ['IFLA_AF_SPEC', {'attrs': [['AF_INET', {'dummy': 65668, 'forwarding': 1, 'mc_forwarding': 0, 'proxy_arp': 0, 'accept_redirects': 1, 'secure_redirects': 1, 'send_redirects': 1, 'shared_media': 1, 'rp_filter': 2, 'accept_source_route': 1, 'bootp_relay': 0, 'log_martians': 0, 'tag': 0, 'arpfilter': 0, 'medium_id': 0, 'noxfrm': 1, 'nopolicy': 1, 'force_igmp_version': 0, 'arp_announce': 0, 'arp_ignore': 0, 'promote_secondaries': 1, 'arp_accept': 0, 'arp_notify': 0, 'accept_local': 0, 'src_vmark': 0, 'proxy_arp_pvlan': 0, 'route_localnet': 0, 'igmpv2_unsolicited_report_interval': 10000, 'igmpv3_unsolicited_report_interval': 1000}], ['AF_INET6', {'attrs': [['IFLA_INET6_FLAGS', 2147483648], ['IFLA_INET6_CACHEINFO', {'max_reasm_len': 65535, 'tstamp': 897729, 'reachable_time': 41896, 'retrans_time': 1000}], ['IFLA_INET6_CONF', {'forwarding': 1, 'hop_limit': 64, 'mtu': 65536, 'accept_ra': 1, 'accept_redirects': 1, 'autoconf': 1, 'dad_transmits': 1, 'router_solicitations': 4294967295, 'router_solicitation_interval': 4000, 'router_solicitation_delay': 1000, 'use_tempaddr': 4294967295, 'temp_valid_lft': 604800, 'temp_preferred_lft': 86400, 'regen_max_retry': 3, 'max_desync_factor': 600, 'max_addresses': 16, 'force_mld_version': 0, 'accept_ra_defrtr': 1, 'accept_ra_pinfo': 1, 'accept_ra_rtr_pref': 1, 'router_probe_interval': 60000, 'accept_ra_rt_info_max_plen': 0, 'proxy_ndp': 0, 'optimistic_dad': 0, 'accept_source_route': 0, 'mc_forwarding': 0, 'disable_ipv6': 0, 'accept_dad': 4294967295, 'force_tllao': 0, 'ndisc_notify': 0}], ['IFLA_INET6_STATS', {'num': 37, 'inpkts': 0, 'inoctets': 0, 'indelivers': 0, 'outforwdatagrams': 0, 'outpkts': 0, 'outoctets': 0, 'inhdrerrors': 0, 'intoobigerrors': 0, 'innoroutes': 0, 'inaddrerrors': 0, 'inunknownprotos': 0, 'intruncatedpkts': 0, 'indiscards': 0, 'outdiscards': 0, 'outnoroutes': 0, 'reasmtimeout': 0, 'reasmreqds': 0, 'reasmoks': 0, 'reasmfails': 0, 'fragoks': 0, 'fragfails': 0, 'fragcreates': 0, 'inmcastpkts': 0, 'outmcastpkts': 0, 'inbcastpkts': 0, 'outbcastpkts': 0, 'inmcastoctets': 0, 'outmcastoctets': 0, 'inbcastoctets': 0, 'outbcastoctets': 0, 'csumerrors': 0, 'noectpkts': 0, 'ect1pkts': 0, 'ect0pkts': 0, 'cepkts': 0}], ['IFLA_INET6_ICMP6STATS', {'num': 6, 'inmsgs': 0, 'inerrors': 0, 'outmsgs': 0, 'outerrors': 0, 'csumerrors': 0}], ['IFLA_INET6_TOKEN', '::'], ['IFLA_INET6_ADDR_GEN_MODE', 0]]}]]}]], 'header': {'length': 1316, 'type': 16, 'flags': 2, 'sequence_number': 255, 'pid': 244329, 'error': None, 'target': 'qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f', 'stats': (0, 0, 0)}, 'state': 'up', 'event': 'RTM_NEWLINK'}]) {{(pid=107952) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:42.056475 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.privileged.agent.linux.ip_lib [-] Namespace qrouter-a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f deleted. {{(pid=107097) remove_netns /opt/stack/neutron/neutron/privileged/agent/linux/ip_lib.py:547}} Jul 01 11:14:42.057462 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo.privsep.daemon [-] privsep: reply[3bfe80b1-007d-4199-ac81-34e97b4c0332]: (4, None) {{(pid=107097) _call_back /usr/local/lib/python3.8/dist-packages/oslo_privsep/daemon.py:501}} Jul 01 11:14:42.058056 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron.agent.l3.agent [-] Router info a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f delete action done, and it was removed from cache. {{(pid=106231) _router_removed /opt/stack/neutron/neutron/agent/l3/agent.py:550}} Jul 01 11:14:42.059038 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG neutron_lib.callbacks.manager [-] Publish callbacks ['neutron.agent.l3.agent.AgentMixin._delete_router-2095906', 'neutron.agent.linux.pd.remove_router-8767249872330'] for router (a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f), after_delete {{(pid=106231) _notify_loop /usr/local/lib/python3.8/dist-packages/neutron_lib/callbacks/manager.py:176}} Jul 01 11:14:42.059562 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" acquired by "neutron.agent.linux.pd.remove_router" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:42.059976 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "l3-agent-pd" "released" by "neutron.agent.linux.pd.remove_router" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:42.060797 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" acquired by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:14:42.061368 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "resize_greenpool" "released" by "neutron.agent.l3.agent.L3NATAgent._resize_process_pool" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:14:42.063063 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: INFO neutron.agent.l3.agent [-] Finished a router delete for a94be6d2-9d84-4e9e-bbdf-d0c00a700d9f, update_id d53a3466-d95a-489b-b688-de77e447dc31. Time elapsed: 0.135 Jul 01 11:14:55.414181 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:15:15.177919 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:15:15.178610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:15:15.178610 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:15:15.178825 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:15:35.417191 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:16:15.178411 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:16:15.179022 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:16:15.179440 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:16:15.179832 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:16:15.417337 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:16:55.418749 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:17:15.179331 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:17:15.180175 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:17:15.180523 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:17:15.180757 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:17:35.420501 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:18:15.179732 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:18:15.180459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:18:15.180459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:18:15.180459 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:18:15.421656 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:18:55.423305 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:19:15.180052 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:19:15.180755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:19:15.180755 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:19:15.182025 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:19:35.426639 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}} Jul 01 11:20:15.181414 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:20:15.182110 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:20:15.182261 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: waited 0.000s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:386}} Jul 01 11:20:15.182652 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" "released" by "neutron.agent.linux.external_process.ProcessMonitor._check_child_processes" :: held 0.001s {{(pid=106231) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:400}} Jul 01 11:20:15.428525 dv-neutron-848393-1-hyperv neutron-l3-agent[106231]: DEBUG oslo_service.periodic_task [None req-c9b58107-493f-4a91-9502-f7b7ccfbb0c3 None None] Running periodic task L3NATAgentWithStateReport.periodic_sync_routers_task {{(pid=106231) run_periodic_tasks /usr/local/lib/python3.8/dist-packages/oslo_service/periodic_task.py:210}}